2024-11-24 22:14:22,038 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-24 22:14:22,052 main DEBUG Took 0.011591 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-24 22:14:22,052 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-24 22:14:22,052 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-24 22:14:22,053 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-24 22:14:22,055 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 22:14:22,063 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-24 22:14:22,077 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 22:14:22,078 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 22:14:22,079 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 22:14:22,079 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 22:14:22,080 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 22:14:22,080 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 22:14:22,082 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 22:14:22,082 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 22:14:22,083 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 22:14:22,083 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 22:14:22,084 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 22:14:22,084 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 22:14:22,085 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 22:14:22,085 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 22:14:22,086 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 22:14:22,086 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 22:14:22,087 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 22:14:22,087 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 22:14:22,088 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 22:14:22,088 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 22:14:22,089 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 22:14:22,089 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 22:14:22,090 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 22:14:22,090 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 22:14:22,091 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 22:14:22,091 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-24 22:14:22,093 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 22:14:22,094 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-24 22:14:22,096 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-24 22:14:22,097 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-24 22:14:22,098 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-24 22:14:22,099 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-24 22:14:22,107 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-24 22:14:22,110 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-24 22:14:22,111 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-24 22:14:22,112 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-24 22:14:22,112 main DEBUG createAppenders(={Console}) 2024-11-24 22:14:22,113 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 initialized 2024-11-24 22:14:22,113 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-24 22:14:22,114 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 OK. 2024-11-24 22:14:22,114 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-24 22:14:22,115 main DEBUG OutputStream closed 2024-11-24 22:14:22,115 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-24 22:14:22,116 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-24 22:14:22,116 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@c88a337 OK 2024-11-24 22:14:22,191 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-24 22:14:22,192 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-24 22:14:22,193 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-24 22:14:22,194 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-24 22:14:22,195 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-24 22:14:22,195 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-24 22:14:22,195 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-24 22:14:22,195 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-24 22:14:22,196 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-24 22:14:22,196 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-24 22:14:22,196 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-24 22:14:22,197 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-24 22:14:22,197 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-24 22:14:22,197 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-24 22:14:22,197 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-24 22:14:22,198 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-24 22:14:22,198 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-24 22:14:22,199 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-24 22:14:22,201 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24 22:14:22,201 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@13330ac6) with optional ClassLoader: null 2024-11-24 22:14:22,201 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-24 22:14:22,202 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@13330ac6] started OK. 2024-11-24T22:14:22,215 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP timeout: 13 mins 2024-11-24 22:14:22,218 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-24 22:14:22,218 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24T22:14:22,471 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/aab95e43-1fd7-94ef-0833-6e5f40c1907f 2024-11-24T22:14:22,491 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$HMasterForTest, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-24T22:14:22,511 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/aab95e43-1fd7-94ef-0833-6e5f40c1907f/cluster_a0f7bd03-a211-96c1-eb4d-b699da328125, deleteOnExit=true 2024-11-24T22:14:22,512 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-24T22:14:22,513 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/aab95e43-1fd7-94ef-0833-6e5f40c1907f/test.cache.data in system properties and HBase conf 2024-11-24T22:14:22,514 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/aab95e43-1fd7-94ef-0833-6e5f40c1907f/hadoop.tmp.dir in system properties and HBase conf 2024-11-24T22:14:22,514 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/aab95e43-1fd7-94ef-0833-6e5f40c1907f/hadoop.log.dir in system properties and HBase conf 2024-11-24T22:14:22,515 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/aab95e43-1fd7-94ef-0833-6e5f40c1907f/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-24T22:14:22,516 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/aab95e43-1fd7-94ef-0833-6e5f40c1907f/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-24T22:14:22,516 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-24T22:14:22,612 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-24T22:14:22,701 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T22:14:22,704 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/aab95e43-1fd7-94ef-0833-6e5f40c1907f/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-24T22:14:22,705 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/aab95e43-1fd7-94ef-0833-6e5f40c1907f/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-24T22:14:22,705 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/aab95e43-1fd7-94ef-0833-6e5f40c1907f/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-24T22:14:22,706 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/aab95e43-1fd7-94ef-0833-6e5f40c1907f/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-24T22:14:22,706 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/aab95e43-1fd7-94ef-0833-6e5f40c1907f/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-24T22:14:22,706 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/aab95e43-1fd7-94ef-0833-6e5f40c1907f/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-24T22:14:22,707 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/aab95e43-1fd7-94ef-0833-6e5f40c1907f/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-24T22:14:22,707 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/aab95e43-1fd7-94ef-0833-6e5f40c1907f/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-24T22:14:22,708 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/aab95e43-1fd7-94ef-0833-6e5f40c1907f/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-24T22:14:22,708 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/aab95e43-1fd7-94ef-0833-6e5f40c1907f/nfs.dump.dir in system properties and HBase conf 2024-11-24T22:14:22,708 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/aab95e43-1fd7-94ef-0833-6e5f40c1907f/java.io.tmpdir in system properties and HBase conf 2024-11-24T22:14:22,709 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/aab95e43-1fd7-94ef-0833-6e5f40c1907f/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-24T22:14:22,709 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/aab95e43-1fd7-94ef-0833-6e5f40c1907f/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-24T22:14:22,710 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/aab95e43-1fd7-94ef-0833-6e5f40c1907f/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-24T22:14:23,550 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-24T22:14:23,644 INFO [Time-limited test {}] log.Log(170): Logging initialized @2289ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-24T22:14:23,728 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T22:14:23,812 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T22:14:23,934 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T22:14:23,934 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T22:14:23,936 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-24T22:14:23,961 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T22:14:23,987 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4d9452bf{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/aab95e43-1fd7-94ef-0833-6e5f40c1907f/hadoop.log.dir/,AVAILABLE} 2024-11-24T22:14:23,989 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5d5b118c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T22:14:24,377 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3260f08c{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/aab95e43-1fd7-94ef-0833-6e5f40c1907f/java.io.tmpdir/jetty-localhost-43395-hadoop-hdfs-3_4_1-tests_jar-_-any-13992110069270711712/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-24T22:14:24,401 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@369f03e8{HTTP/1.1, (http/1.1)}{localhost:43395} 2024-11-24T22:14:24,402 INFO [Time-limited test {}] server.Server(415): Started @3047ms 2024-11-24T22:14:25,039 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T22:14:25,048 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T22:14:25,071 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T22:14:25,071 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T22:14:25,071 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-24T22:14:25,073 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@20ec9bac{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/aab95e43-1fd7-94ef-0833-6e5f40c1907f/hadoop.log.dir/,AVAILABLE} 2024-11-24T22:14:25,074 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5026a795{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T22:14:25,265 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4f33137f{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/aab95e43-1fd7-94ef-0833-6e5f40c1907f/java.io.tmpdir/jetty-localhost-40115-hadoop-hdfs-3_4_1-tests_jar-_-any-13037904335030880965/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T22:14:25,266 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6de9cba2{HTTP/1.1, (http/1.1)}{localhost:40115} 2024-11-24T22:14:25,266 INFO [Time-limited test {}] server.Server(415): Started @3912ms 2024-11-24T22:14:25,336 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-24T22:14:25,686 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T22:14:25,698 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T22:14:25,724 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T22:14:25,724 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T22:14:25,724 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-24T22:14:25,737 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5b3caf0c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/aab95e43-1fd7-94ef-0833-6e5f40c1907f/hadoop.log.dir/,AVAILABLE} 2024-11-24T22:14:25,738 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@540e5e4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T22:14:25,976 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7549e2f1{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/aab95e43-1fd7-94ef-0833-6e5f40c1907f/java.io.tmpdir/jetty-localhost-45093-hadoop-hdfs-3_4_1-tests_jar-_-any-16505627737584288401/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T22:14:25,980 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@53e0679{HTTP/1.1, (http/1.1)}{localhost:45093} 2024-11-24T22:14:25,980 INFO [Time-limited test {}] server.Server(415): Started @4626ms 2024-11-24T22:14:25,988 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-24T22:14:26,161 WARN [Thread-95 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/aab95e43-1fd7-94ef-0833-6e5f40c1907f/cluster_a0f7bd03-a211-96c1-eb4d-b699da328125/data/data2/current/BP-894544699-172.17.0.2-1732486463304/current, will proceed with Du for space computation calculation, 2024-11-24T22:14:26,161 WARN [Thread-94 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/aab95e43-1fd7-94ef-0833-6e5f40c1907f/cluster_a0f7bd03-a211-96c1-eb4d-b699da328125/data/data1/current/BP-894544699-172.17.0.2-1732486463304/current, will proceed with Du for space computation calculation, 2024-11-24T22:14:26,180 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T22:14:26,200 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T22:14:26,260 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T22:14:26,260 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T22:14:26,261 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-24T22:14:26,281 WARN [Thread-109 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/aab95e43-1fd7-94ef-0833-6e5f40c1907f/cluster_a0f7bd03-a211-96c1-eb4d-b699da328125/data/data3/current/BP-894544699-172.17.0.2-1732486463304/current, will proceed with Du for space computation calculation, 2024-11-24T22:14:26,284 WARN [Thread-112 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/aab95e43-1fd7-94ef-0833-6e5f40c1907f/cluster_a0f7bd03-a211-96c1-eb4d-b699da328125/data/data4/current/BP-894544699-172.17.0.2-1732486463304/current, will proceed with Du for space computation calculation, 2024-11-24T22:14:26,294 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@469f5a42{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/aab95e43-1fd7-94ef-0833-6e5f40c1907f/hadoop.log.dir/,AVAILABLE} 2024-11-24T22:14:26,295 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@75445ba5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T22:14:26,360 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-24T22:14:26,430 WARN [Thread-83 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-24T22:14:26,489 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3f5a320f99dd41df with lease ID 0x1506af56628f6d46: Processing first storage report for DS-56b705de-da8a-4837-9f73-0e8d9391298c from datanode DatanodeRegistration(127.0.0.1:35383, datanodeUuid=bcf2205b-d782-4d32-8ddc-4ee412a98f8f, infoPort=38755, infoSecurePort=0, ipcPort=46087, storageInfo=lv=-57;cid=testClusterID;nsid=1976966782;c=1732486463304) 2024-11-24T22:14:26,490 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3f5a320f99dd41df with lease ID 0x1506af56628f6d46: from storage DS-56b705de-da8a-4837-9f73-0e8d9391298c node DatanodeRegistration(127.0.0.1:35383, datanodeUuid=bcf2205b-d782-4d32-8ddc-4ee412a98f8f, infoPort=38755, infoSecurePort=0, ipcPort=46087, storageInfo=lv=-57;cid=testClusterID;nsid=1976966782;c=1732486463304), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-24T22:14:26,491 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe4221389d0ff5eda with lease ID 0x1506af56628f6d47: Processing first storage report for DS-e98687a1-55aa-42d3-a359-80f43904e747 from datanode DatanodeRegistration(127.0.0.1:34013, datanodeUuid=9e82a45a-763a-4117-8e80-211b5d4bffb5, infoPort=36251, infoSecurePort=0, ipcPort=36463, storageInfo=lv=-57;cid=testClusterID;nsid=1976966782;c=1732486463304) 2024-11-24T22:14:26,491 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe4221389d0ff5eda with lease ID 0x1506af56628f6d47: from storage DS-e98687a1-55aa-42d3-a359-80f43904e747 node DatanodeRegistration(127.0.0.1:34013, datanodeUuid=9e82a45a-763a-4117-8e80-211b5d4bffb5, infoPort=36251, infoSecurePort=0, ipcPort=36463, storageInfo=lv=-57;cid=testClusterID;nsid=1976966782;c=1732486463304), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-24T22:14:26,492 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe4221389d0ff5eda with lease ID 0x1506af56628f6d47: Processing first storage report for DS-d0b7785b-b032-4551-8b59-4f96b03a9a70 from datanode DatanodeRegistration(127.0.0.1:34013, datanodeUuid=9e82a45a-763a-4117-8e80-211b5d4bffb5, infoPort=36251, infoSecurePort=0, ipcPort=36463, storageInfo=lv=-57;cid=testClusterID;nsid=1976966782;c=1732486463304) 2024-11-24T22:14:26,492 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe4221389d0ff5eda with lease ID 0x1506af56628f6d47: from storage DS-d0b7785b-b032-4551-8b59-4f96b03a9a70 node DatanodeRegistration(127.0.0.1:34013, datanodeUuid=9e82a45a-763a-4117-8e80-211b5d4bffb5, infoPort=36251, infoSecurePort=0, ipcPort=36463, storageInfo=lv=-57;cid=testClusterID;nsid=1976966782;c=1732486463304), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-24T22:14:26,493 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3f5a320f99dd41df with lease ID 0x1506af56628f6d46: Processing first storage report for DS-a9e76dd6-7914-4cb0-a009-756cbdde5918 from datanode DatanodeRegistration(127.0.0.1:35383, datanodeUuid=bcf2205b-d782-4d32-8ddc-4ee412a98f8f, infoPort=38755, infoSecurePort=0, ipcPort=46087, storageInfo=lv=-57;cid=testClusterID;nsid=1976966782;c=1732486463304) 2024-11-24T22:14:26,493 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3f5a320f99dd41df with lease ID 0x1506af56628f6d46: from storage DS-a9e76dd6-7914-4cb0-a009-756cbdde5918 node DatanodeRegistration(127.0.0.1:35383, datanodeUuid=bcf2205b-d782-4d32-8ddc-4ee412a98f8f, infoPort=38755, infoSecurePort=0, ipcPort=46087, storageInfo=lv=-57;cid=testClusterID;nsid=1976966782;c=1732486463304), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-24T22:14:26,543 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6eecab6b{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/aab95e43-1fd7-94ef-0833-6e5f40c1907f/java.io.tmpdir/jetty-localhost-38703-hadoop-hdfs-3_4_1-tests_jar-_-any-10691711336528090704/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T22:14:26,544 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3fb3fd4b{HTTP/1.1, (http/1.1)}{localhost:38703} 2024-11-24T22:14:26,549 INFO [Time-limited test {}] server.Server(415): Started @5194ms 2024-11-24T22:14:26,554 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-24T22:14:26,821 WARN [Thread-139 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/aab95e43-1fd7-94ef-0833-6e5f40c1907f/cluster_a0f7bd03-a211-96c1-eb4d-b699da328125/data/data5/current/BP-894544699-172.17.0.2-1732486463304/current, will proceed with Du for space computation calculation, 2024-11-24T22:14:26,841 WARN [Thread-140 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/aab95e43-1fd7-94ef-0833-6e5f40c1907f/cluster_a0f7bd03-a211-96c1-eb4d-b699da328125/data/data6/current/BP-894544699-172.17.0.2-1732486463304/current, will proceed with Du for space computation calculation, 2024-11-24T22:14:26,965 WARN [Thread-129 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-24T22:14:26,984 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x59897513e0a70984 with lease ID 0x1506af56628f6d48: Processing first storage report for DS-fb965c9f-83ab-43e9-b4f1-51c95ecaf7db from datanode DatanodeRegistration(127.0.0.1:34899, datanodeUuid=00cd84f3-3e98-4d05-974d-4d5adbfbfbc2, infoPort=42065, infoSecurePort=0, ipcPort=32861, storageInfo=lv=-57;cid=testClusterID;nsid=1976966782;c=1732486463304) 2024-11-24T22:14:26,985 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x59897513e0a70984 with lease ID 0x1506af56628f6d48: from storage DS-fb965c9f-83ab-43e9-b4f1-51c95ecaf7db node DatanodeRegistration(127.0.0.1:34899, datanodeUuid=00cd84f3-3e98-4d05-974d-4d5adbfbfbc2, infoPort=42065, infoSecurePort=0, ipcPort=32861, storageInfo=lv=-57;cid=testClusterID;nsid=1976966782;c=1732486463304), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-24T22:14:26,985 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x59897513e0a70984 with lease ID 0x1506af56628f6d48: Processing first storage report for DS-c5ac8421-4ced-48f1-8436-ef338195ed35 from datanode DatanodeRegistration(127.0.0.1:34899, datanodeUuid=00cd84f3-3e98-4d05-974d-4d5adbfbfbc2, infoPort=42065, infoSecurePort=0, ipcPort=32861, storageInfo=lv=-57;cid=testClusterID;nsid=1976966782;c=1732486463304) 2024-11-24T22:14:26,985 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x59897513e0a70984 with lease ID 0x1506af56628f6d48: from storage DS-c5ac8421-4ced-48f1-8436-ef338195ed35 node DatanodeRegistration(127.0.0.1:34899, datanodeUuid=00cd84f3-3e98-4d05-974d-4d5adbfbfbc2, infoPort=42065, infoSecurePort=0, ipcPort=32861, storageInfo=lv=-57;cid=testClusterID;nsid=1976966782;c=1732486463304), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-24T22:14:27,313 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/aab95e43-1fd7-94ef-0833-6e5f40c1907f 2024-11-24T22:14:27,363 DEBUG [Time-limited test {}] zookeeper.MiniZooKeeperCluster(246): Failed binding ZK Server to client port: 56582 java.net.BindException: Address already in use at sun.nio.ch.Net.bind0(Native Method) ~[?:?] at sun.nio.ch.Net.bind(Net.java:555) ~[?:?] at sun.nio.ch.ServerSocketChannelImpl.netBind(ServerSocketChannelImpl.java:337) ~[?:?] at sun.nio.ch.ServerSocketChannelImpl.bind(ServerSocketChannelImpl.java:294) ~[?:?] at sun.nio.ch.ServerSocketAdaptor.bind(ServerSocketAdaptor.java:89) ~[?:?] at sun.nio.ch.ServerSocketAdaptor.bind(ServerSocketAdaptor.java:81) ~[?:?] at org.apache.zookeeper.server.NIOServerCnxnFactory.configure(NIOServerCnxnFactory.java:662) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.server.ServerCnxnFactory.configure(ServerCnxnFactory.java:109) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.server.ServerCnxnFactory.configure(ServerCnxnFactory.java:105) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster.startup(MiniZooKeeperCluster.java:242) ~[hbase-zookeeper-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseZKTestingUtil.startMiniZKCluster(HBaseZKTestingUtil.java:131) ~[hbase-zookeeper-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseZKTestingUtil.startMiniZKCluster(HBaseZKTestingUtil.java:104) ~[hbase-zookeeper-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.startMiniCluster(HBaseTestingUtil.java:826) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.setUpBeforeClass(TestRollbackSCP.java:124) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.invokeMethod(RunBefores.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:24) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:27,449 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/aab95e43-1fd7-94ef-0833-6e5f40c1907f/cluster_a0f7bd03-a211-96c1-eb4d-b699da328125/zookeeper_0, clientPort=56583, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/aab95e43-1fd7-94ef-0833-6e5f40c1907f/cluster_a0f7bd03-a211-96c1-eb4d-b699da328125/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/aab95e43-1fd7-94ef-0833-6e5f40c1907f/cluster_a0f7bd03-a211-96c1-eb4d-b699da328125/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-24T22:14:27,462 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=56583 2024-11-24T22:14:27,485 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T22:14:27,499 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T22:14:27,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34013 is added to blk_1073741825_1001 (size=7) 2024-11-24T22:14:27,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35383 is added to blk_1073741825_1001 (size=7) 2024-11-24T22:14:27,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741825_1001 (size=7) 2024-11-24T22:14:27,990 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c with version=8 2024-11-24T22:14:27,991 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/hbase-staging 2024-11-24T22:14:28,175 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-24T22:14:28,514 INFO [Time-limited test {}] client.ConnectionUtils(128): master/bd6592c37bb2:0 server-side Connection retries=45 2024-11-24T22:14:28,526 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T22:14:28,527 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T22:14:28,533 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T22:14:28,533 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T22:14:28,533 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T22:14:28,757 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-24T22:14:28,854 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-24T22:14:28,870 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-24T22:14:28,875 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T22:14:28,932 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 7523 (auto-detected) 2024-11-24T22:14:28,933 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-24T22:14:28,971 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36279 2024-11-24T22:14:29,006 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36279 connecting to ZooKeeper ensemble=127.0.0.1:56583 2024-11-24T22:14:29,063 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:362790x0, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T22:14:29,069 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36279-0x1014c0b79ef0000 connected 2024-11-24T22:14:29,112 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T22:14:29,116 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T22:14:29,130 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36279-0x1014c0b79ef0000, quorum=127.0.0.1:56583, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T22:14:29,134 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c, hbase.cluster.distributed=false 2024-11-24T22:14:29,161 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36279-0x1014c0b79ef0000, quorum=127.0.0.1:56583, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T22:14:29,169 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36279 2024-11-24T22:14:29,170 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36279 2024-11-24T22:14:29,180 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36279 2024-11-24T22:14:29,181 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36279 2024-11-24T22:14:29,182 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36279 2024-11-24T22:14:29,313 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/bd6592c37bb2:0 server-side Connection retries=45 2024-11-24T22:14:29,314 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T22:14:29,315 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T22:14:29,315 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T22:14:29,315 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T22:14:29,315 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T22:14:29,319 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T22:14:29,321 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T22:14:29,322 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41619 2024-11-24T22:14:29,324 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41619 connecting to ZooKeeper ensemble=127.0.0.1:56583 2024-11-24T22:14:29,325 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T22:14:29,337 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T22:14:29,343 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:416190x0, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T22:14:29,345 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:416190x0, quorum=127.0.0.1:56583, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T22:14:29,346 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41619-0x1014c0b79ef0001 connected 2024-11-24T22:14:29,350 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T22:14:29,364 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T22:14:29,366 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41619-0x1014c0b79ef0001, quorum=127.0.0.1:56583, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T22:14:29,373 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41619-0x1014c0b79ef0001, quorum=127.0.0.1:56583, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T22:14:29,373 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41619 2024-11-24T22:14:29,374 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41619 2024-11-24T22:14:29,375 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41619 2024-11-24T22:14:29,380 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41619 2024-11-24T22:14:29,381 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41619 2024-11-24T22:14:29,406 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/bd6592c37bb2:0 server-side Connection retries=45 2024-11-24T22:14:29,406 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T22:14:29,407 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T22:14:29,407 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T22:14:29,407 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T22:14:29,408 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T22:14:29,408 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T22:14:29,408 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T22:14:29,410 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38125 2024-11-24T22:14:29,414 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38125 connecting to ZooKeeper ensemble=127.0.0.1:56583 2024-11-24T22:14:29,415 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T22:14:29,418 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T22:14:29,437 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:381250x0, quorum=127.0.0.1:56583, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T22:14:29,438 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T22:14:29,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:381250x0, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T22:14:29,441 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38125-0x1014c0b79ef0002 connected 2024-11-24T22:14:29,444 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T22:14:29,446 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38125-0x1014c0b79ef0002, quorum=127.0.0.1:56583, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T22:14:29,448 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38125-0x1014c0b79ef0002, quorum=127.0.0.1:56583, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T22:14:29,449 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38125 2024-11-24T22:14:29,450 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38125 2024-11-24T22:14:29,452 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38125 2024-11-24T22:14:29,455 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38125 2024-11-24T22:14:29,456 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38125 2024-11-24T22:14:29,476 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/bd6592c37bb2:0 server-side Connection retries=45 2024-11-24T22:14:29,477 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T22:14:29,477 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T22:14:29,477 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T22:14:29,477 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T22:14:29,477 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T22:14:29,478 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T22:14:29,478 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T22:14:29,479 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42639 2024-11-24T22:14:29,481 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42639 connecting to ZooKeeper ensemble=127.0.0.1:56583 2024-11-24T22:14:29,483 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T22:14:29,486 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T22:14:29,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:426390x0, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T22:14:29,497 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:426390x0, quorum=127.0.0.1:56583, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T22:14:29,498 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T22:14:29,500 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42639-0x1014c0b79ef0003 connected 2024-11-24T22:14:29,503 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T22:14:29,504 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42639-0x1014c0b79ef0003, quorum=127.0.0.1:56583, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T22:14:29,509 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42639-0x1014c0b79ef0003, quorum=127.0.0.1:56583, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T22:14:29,512 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42639 2024-11-24T22:14:29,513 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42639 2024-11-24T22:14:29,513 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42639 2024-11-24T22:14:29,514 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42639 2024-11-24T22:14:29,514 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42639 2024-11-24T22:14:29,530 DEBUG [M:0;bd6592c37bb2:36279 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;bd6592c37bb2:36279 2024-11-24T22:14:29,533 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/bd6592c37bb2,36279,1732486468255 2024-11-24T22:14:29,542 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41619-0x1014c0b79ef0001, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T22:14:29,542 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38125-0x1014c0b79ef0002, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T22:14:29,542 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42639-0x1014c0b79ef0003, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T22:14:29,543 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36279-0x1014c0b79ef0000, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T22:14:29,546 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36279-0x1014c0b79ef0000, quorum=127.0.0.1:56583, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/bd6592c37bb2,36279,1732486468255 2024-11-24T22:14:29,582 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38125-0x1014c0b79ef0002, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T22:14:29,582 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36279-0x1014c0b79ef0000, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T22:14:29,583 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38125-0x1014c0b79ef0002, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T22:14:29,583 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41619-0x1014c0b79ef0001, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T22:14:29,583 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41619-0x1014c0b79ef0001, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T22:14:29,583 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42639-0x1014c0b79ef0003, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T22:14:29,583 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42639-0x1014c0b79ef0003, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T22:14:29,586 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36279-0x1014c0b79ef0000, quorum=127.0.0.1:56583, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-24T22:14:29,591 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/bd6592c37bb2,36279,1732486468255 from backup master directory 2024-11-24T22:14:29,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42639-0x1014c0b79ef0003, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T22:14:29,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36279-0x1014c0b79ef0000, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/bd6592c37bb2,36279,1732486468255 2024-11-24T22:14:29,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41619-0x1014c0b79ef0001, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T22:14:29,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38125-0x1014c0b79ef0002, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T22:14:29,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36279-0x1014c0b79ef0000, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T22:14:29,596 WARN [master/bd6592c37bb2:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T22:14:29,596 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=bd6592c37bb2,36279,1732486468255 2024-11-24T22:14:29,599 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-24T22:14:29,605 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-24T22:14:29,686 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/hbase.id] with ID: 905a4a1c-dd04-42c5-bf3f-b8b8bda6274a 2024-11-24T22:14:29,686 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/.tmp/hbase.id 2024-11-24T22:14:29,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34013 is added to blk_1073741826_1002 (size=42) 2024-11-24T22:14:29,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741826_1002 (size=42) 2024-11-24T22:14:29,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35383 is added to blk_1073741826_1002 (size=42) 2024-11-24T22:14:29,709 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/.tmp/hbase.id]:[hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/hbase.id] 2024-11-24T22:14:29,789 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T22:14:29,795 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-24T22:14:29,821 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 23ms. 2024-11-24T22:14:29,825 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41619-0x1014c0b79ef0001, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T22:14:29,825 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42639-0x1014c0b79ef0003, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T22:14:29,826 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38125-0x1014c0b79ef0002, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T22:14:29,826 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36279-0x1014c0b79ef0000, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T22:14:29,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34013 is added to blk_1073741827_1003 (size=196) 2024-11-24T22:14:29,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35383 is added to blk_1073741827_1003 (size=196) 2024-11-24T22:14:29,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741827_1003 (size=196) 2024-11-24T22:14:29,895 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T22:14:29,897 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-24T22:14:29,919 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:29,933 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T22:14:30,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741828_1004 (size=1189) 2024-11-24T22:14:30,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35383 is added to blk_1073741828_1004 (size=1189) 2024-11-24T22:14:30,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34013 is added to blk_1073741828_1004 (size=1189) 2024-11-24T22:14:30,033 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/MasterData/data/master/store 2024-11-24T22:14:30,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34013 is added to blk_1073741829_1005 (size=34) 2024-11-24T22:14:30,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741829_1005 (size=34) 2024-11-24T22:14:30,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35383 is added to blk_1073741829_1005 (size=34) 2024-11-24T22:14:30,091 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-24T22:14:30,095 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:30,096 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T22:14:30,096 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T22:14:30,096 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T22:14:30,098 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T22:14:30,098 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T22:14:30,098 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T22:14:30,099 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732486470096Disabling compacts and flushes for region at 1732486470096Disabling writes for close at 1732486470098 (+2 ms)Writing region close event to WAL at 1732486470098Closed at 1732486470098 2024-11-24T22:14:30,103 WARN [master/bd6592c37bb2:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/MasterData/data/master/store/.initializing 2024-11-24T22:14:30,104 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/MasterData/WALs/bd6592c37bb2,36279,1732486468255 2024-11-24T22:14:30,114 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T22:14:30,134 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=bd6592c37bb2%2C36279%2C1732486468255, suffix=, logDir=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/MasterData/WALs/bd6592c37bb2,36279,1732486468255, archiveDir=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/MasterData/oldWALs, maxLogs=10 2024-11-24T22:14:30,167 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/MasterData/WALs/bd6592c37bb2,36279,1732486468255/bd6592c37bb2%2C36279%2C1732486468255.1732486470140, exclude list is [], retry=0 2024-11-24T22:14:30,187 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34899,DS-fb965c9f-83ab-43e9-b4f1-51c95ecaf7db,DISK] 2024-11-24T22:14:30,187 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34013,DS-e98687a1-55aa-42d3-a359-80f43904e747,DISK] 2024-11-24T22:14:30,187 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35383,DS-56b705de-da8a-4837-9f73-0e8d9391298c,DISK] 2024-11-24T22:14:30,191 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-24T22:14:30,242 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/MasterData/WALs/bd6592c37bb2,36279,1732486468255/bd6592c37bb2%2C36279%2C1732486468255.1732486470140 2024-11-24T22:14:30,243 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42065:42065),(127.0.0.1/127.0.0.1:38755:38755),(127.0.0.1/127.0.0.1:36251:36251)] 2024-11-24T22:14:30,244 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-24T22:14:30,244 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:30,248 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T22:14:30,249 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T22:14:30,292 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T22:14:30,331 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-24T22:14:30,336 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:30,340 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T22:14:30,341 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T22:14:30,344 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-24T22:14:30,344 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:30,346 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:14:30,346 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T22:14:30,349 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-24T22:14:30,349 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:30,350 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:14:30,350 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T22:14:30,354 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-24T22:14:30,354 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:30,355 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:14:30,355 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T22:14:30,359 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T22:14:30,360 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T22:14:30,365 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T22:14:30,366 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T22:14:30,369 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T22:14:30,373 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T22:14:30,379 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T22:14:30,380 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64181870, jitterRate=-0.04361560940742493}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T22:14:30,387 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732486470262Initializing all the Stores at 1732486470266 (+4 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732486470266Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486470267 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486470267Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486470267Cleaning up temporary data from old regions at 1732486470366 (+99 ms)Region opened successfully at 1732486470387 (+21 ms) 2024-11-24T22:14:30,388 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-24T22:14:30,438 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39e50855, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=bd6592c37bb2/172.17.0.2:0 2024-11-24T22:14:30,481 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-24T22:14:30,496 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T22:14:30,496 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T22:14:30,500 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T22:14:30,501 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-24T22:14:30,506 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-24T22:14:30,506 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-24T22:14:30,536 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-24T22:14:30,548 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36279-0x1014c0b79ef0000, quorum=127.0.0.1:56583, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-24T22:14:30,554 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-24T22:14:30,558 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-24T22:14:30,559 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36279-0x1014c0b79ef0000, quorum=127.0.0.1:56583, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-24T22:14:30,561 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-24T22:14:30,563 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-24T22:14:30,567 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36279-0x1014c0b79ef0000, quorum=127.0.0.1:56583, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-24T22:14:30,568 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-24T22:14:30,570 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36279-0x1014c0b79ef0000, quorum=127.0.0.1:56583, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-24T22:14:30,572 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-24T22:14:30,595 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36279-0x1014c0b79ef0000, quorum=127.0.0.1:56583, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-24T22:14:30,598 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-24T22:14:30,602 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36279-0x1014c0b79ef0000, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T22:14:30,602 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38125-0x1014c0b79ef0002, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T22:14:30,603 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38125-0x1014c0b79ef0002, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T22:14:30,603 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36279-0x1014c0b79ef0000, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T22:14:30,603 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42639-0x1014c0b79ef0003, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T22:14:30,603 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41619-0x1014c0b79ef0001, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T22:14:30,603 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41619-0x1014c0b79ef0001, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T22:14:30,604 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42639-0x1014c0b79ef0003, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T22:14:30,608 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=bd6592c37bb2,36279,1732486468255, sessionid=0x1014c0b79ef0000, setting cluster-up flag (Was=false) 2024-11-24T22:14:30,623 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38125-0x1014c0b79ef0002, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T22:14:30,623 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36279-0x1014c0b79ef0000, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T22:14:30,623 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42639-0x1014c0b79ef0003, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T22:14:30,623 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41619-0x1014c0b79ef0001, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T22:14:30,632 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-24T22:14:30,634 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=bd6592c37bb2,36279,1732486468255 2024-11-24T22:14:30,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41619-0x1014c0b79ef0001, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T22:14:30,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36279-0x1014c0b79ef0000, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T22:14:30,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38125-0x1014c0b79ef0002, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T22:14:30,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42639-0x1014c0b79ef0003, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T22:14:30,647 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-24T22:14:30,649 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=bd6592c37bb2,36279,1732486468255 2024-11-24T22:14:30,656 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-24T22:14:30,721 INFO [RS:2;bd6592c37bb2:42639 {}] regionserver.HRegionServer(746): ClusterId : 905a4a1c-dd04-42c5-bf3f-b8b8bda6274a 2024-11-24T22:14:30,725 DEBUG [RS:2;bd6592c37bb2:42639 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T22:14:30,726 INFO [RS:1;bd6592c37bb2:38125 {}] regionserver.HRegionServer(746): ClusterId : 905a4a1c-dd04-42c5-bf3f-b8b8bda6274a 2024-11-24T22:14:30,726 DEBUG [RS:1;bd6592c37bb2:38125 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T22:14:30,727 INFO [RS:0;bd6592c37bb2:41619 {}] regionserver.HRegionServer(746): ClusterId : 905a4a1c-dd04-42c5-bf3f-b8b8bda6274a 2024-11-24T22:14:30,727 DEBUG [RS:0;bd6592c37bb2:41619 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T22:14:30,736 DEBUG [RS:1;bd6592c37bb2:38125 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T22:14:30,736 DEBUG [RS:1;bd6592c37bb2:38125 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T22:14:30,736 DEBUG [RS:2;bd6592c37bb2:42639 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T22:14:30,736 DEBUG [RS:2;bd6592c37bb2:42639 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T22:14:30,739 DEBUG [RS:0;bd6592c37bb2:41619 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T22:14:30,739 DEBUG [RS:0;bd6592c37bb2:41619 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T22:14:30,740 DEBUG [RS:1;bd6592c37bb2:38125 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T22:14:30,740 DEBUG [RS:1;bd6592c37bb2:38125 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5842e3ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=bd6592c37bb2/172.17.0.2:0 2024-11-24T22:14:30,742 DEBUG [RS:2;bd6592c37bb2:42639 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T22:14:30,742 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T22:14:30,745 DEBUG [RS:0;bd6592c37bb2:41619 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T22:14:30,745 DEBUG [RS:0;bd6592c37bb2:41619 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8b80e50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=bd6592c37bb2/172.17.0.2:0 2024-11-24T22:14:30,746 DEBUG [RS:2;bd6592c37bb2:42639 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52fc34ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=bd6592c37bb2/172.17.0.2:0 2024-11-24T22:14:30,765 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-24T22:14:30,765 DEBUG [RS:1;bd6592c37bb2:38125 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;bd6592c37bb2:38125 2024-11-24T22:14:30,769 INFO [RS:1;bd6592c37bb2:38125 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T22:14:30,769 INFO [RS:1;bd6592c37bb2:38125 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T22:14:30,769 DEBUG [RS:1;bd6592c37bb2:38125 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T22:14:30,773 INFO [RS:1;bd6592c37bb2:38125 {}] regionserver.HRegionServer(2659): reportForDuty to master=bd6592c37bb2,36279,1732486468255 with port=38125, startcode=1732486469405 2024-11-24T22:14:30,774 DEBUG [RS:0;bd6592c37bb2:41619 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;bd6592c37bb2:41619 2024-11-24T22:14:30,774 INFO [RS:0;bd6592c37bb2:41619 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T22:14:30,774 INFO [RS:0;bd6592c37bb2:41619 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T22:14:30,774 DEBUG [RS:0;bd6592c37bb2:41619 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T22:14:30,775 INFO [RS:0;bd6592c37bb2:41619 {}] regionserver.HRegionServer(2659): reportForDuty to master=bd6592c37bb2,36279,1732486468255 with port=41619, startcode=1732486469269 2024-11-24T22:14:30,775 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-24T22:14:30,787 DEBUG [RS:0;bd6592c37bb2:41619 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T22:14:30,787 DEBUG [RS:1;bd6592c37bb2:38125 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T22:14:30,783 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: bd6592c37bb2,36279,1732486468255 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-24T22:14:30,790 DEBUG [RS:2;bd6592c37bb2:42639 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;bd6592c37bb2:42639 2024-11-24T22:14:30,790 INFO [RS:2;bd6592c37bb2:42639 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T22:14:30,790 INFO [RS:2;bd6592c37bb2:42639 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T22:14:30,790 DEBUG [RS:2;bd6592c37bb2:42639 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T22:14:30,791 INFO [RS:2;bd6592c37bb2:42639 {}] regionserver.HRegionServer(2659): reportForDuty to master=bd6592c37bb2,36279,1732486468255 with port=42639, startcode=1732486469476 2024-11-24T22:14:30,792 DEBUG [RS:2;bd6592c37bb2:42639 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T22:14:30,800 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/bd6592c37bb2:0, corePoolSize=5, maxPoolSize=5 2024-11-24T22:14:30,800 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/bd6592c37bb2:0, corePoolSize=5, maxPoolSize=5 2024-11-24T22:14:30,801 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/bd6592c37bb2:0, corePoolSize=5, maxPoolSize=5 2024-11-24T22:14:30,801 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/bd6592c37bb2:0, corePoolSize=5, maxPoolSize=5 2024-11-24T22:14:30,801 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/bd6592c37bb2:0, corePoolSize=10, maxPoolSize=10 2024-11-24T22:14:30,801 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/bd6592c37bb2:0, corePoolSize=1, maxPoolSize=1 2024-11-24T22:14:30,801 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/bd6592c37bb2:0, corePoolSize=2, maxPoolSize=2 2024-11-24T22:14:30,801 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/bd6592c37bb2:0, corePoolSize=1, maxPoolSize=1 2024-11-24T22:14:30,818 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732486500817 2024-11-24T22:14:30,820 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-24T22:14:30,822 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-24T22:14:30,826 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-24T22:14:30,827 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-24T22:14:30,827 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-24T22:14:30,827 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-24T22:14:30,837 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T22:14:30,841 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-24T22:14:30,841 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T22:14:30,842 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-24T22:14:30,843 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-24T22:14:30,843 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-24T22:14:30,856 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-24T22:14:30,857 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-24T22:14:30,860 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:30,861 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-24T22:14:30,863 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/bd6592c37bb2:0:becomeActiveMaster-HFileCleaner.large.0-1732486470860,5,FailOnTimeoutGroup] 2024-11-24T22:14:30,870 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58735, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T22:14:30,872 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36369, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T22:14:30,874 INFO [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53651, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T22:14:30,878 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36279 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:30,884 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/bd6592c37bb2:0:becomeActiveMaster-HFileCleaner.small.0-1732486470873,5,FailOnTimeoutGroup] 2024-11-24T22:14:30,884 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T22:14:30,885 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-24T22:14:30,885 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36279 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:30,886 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-24T22:14:30,886 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36279 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:30,887 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-24T22:14:30,923 DEBUG [RS:1;bd6592c37bb2:38125 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-24T22:14:30,924 WARN [RS:1;bd6592c37bb2:38125 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-24T22:14:30,924 DEBUG [RS:2;bd6592c37bb2:42639 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-24T22:14:30,924 WARN [RS:2;bd6592c37bb2:42639 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-24T22:14:30,925 DEBUG [RS:0;bd6592c37bb2:41619 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-24T22:14:30,925 WARN [RS:0;bd6592c37bb2:41619 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-24T22:14:30,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34013 is added to blk_1073741831_1007 (size=1321) 2024-11-24T22:14:30,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35383 is added to blk_1073741831_1007 (size=1321) 2024-11-24T22:14:30,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741831_1007 (size=1321) 2024-11-24T22:14:30,994 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-24T22:14:30,995 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c 2024-11-24T22:14:31,025 INFO [RS:1;bd6592c37bb2:38125 {}] regionserver.HRegionServer(2659): reportForDuty to master=bd6592c37bb2,36279,1732486468255 with port=38125, startcode=1732486469405 2024-11-24T22:14:31,026 INFO [RS:2;bd6592c37bb2:42639 {}] regionserver.HRegionServer(2659): reportForDuty to master=bd6592c37bb2,36279,1732486468255 with port=42639, startcode=1732486469476 2024-11-24T22:14:31,028 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36279 {}] master.ServerManager(363): Checking decommissioned status of RegionServer bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:31,028 INFO [RS:0;bd6592c37bb2:41619 {}] regionserver.HRegionServer(2659): reportForDuty to master=bd6592c37bb2,36279,1732486468255 with port=41619, startcode=1732486469269 2024-11-24T22:14:31,032 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36279 {}] master.ServerManager(517): Registering regionserver=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:31,052 DEBUG [RS:1;bd6592c37bb2:38125 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c 2024-11-24T22:14:31,053 DEBUG [RS:1;bd6592c37bb2:38125 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36281 2024-11-24T22:14:31,053 DEBUG [RS:1;bd6592c37bb2:38125 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T22:14:31,054 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36279 {}] master.ServerManager(363): Checking decommissioned status of RegionServer bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:31,054 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36279 {}] master.ServerManager(517): Registering regionserver=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:31,058 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36279-0x1014c0b79ef0000, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T22:14:31,059 DEBUG [RS:1;bd6592c37bb2:38125 {}] zookeeper.ZKUtil(111): regionserver:38125-0x1014c0b79ef0002, quorum=127.0.0.1:56583, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:31,059 WARN [RS:1;bd6592c37bb2:38125 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T22:14:31,060 INFO [RS:1;bd6592c37bb2:38125 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T22:14:31,060 DEBUG [RS:1;bd6592c37bb2:38125 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/WALs/bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:31,061 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36279 {}] master.ServerManager(363): Checking decommissioned status of RegionServer bd6592c37bb2,41619,1732486469269 2024-11-24T22:14:31,061 DEBUG [RS:2;bd6592c37bb2:42639 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c 2024-11-24T22:14:31,061 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36279 {}] master.ServerManager(517): Registering regionserver=bd6592c37bb2,41619,1732486469269 2024-11-24T22:14:31,061 DEBUG [RS:2;bd6592c37bb2:42639 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36281 2024-11-24T22:14:31,061 DEBUG [RS:2;bd6592c37bb2:42639 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T22:14:31,070 DEBUG [RS:2;bd6592c37bb2:42639 {}] zookeeper.ZKUtil(111): regionserver:42639-0x1014c0b79ef0003, quorum=127.0.0.1:56583, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:31,070 WARN [RS:2;bd6592c37bb2:42639 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T22:14:31,070 INFO [RS:2;bd6592c37bb2:42639 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T22:14:31,070 DEBUG [RS:2;bd6592c37bb2:42639 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/WALs/bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:31,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741832_1008 (size=32) 2024-11-24T22:14:31,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35383 is added to blk_1073741832_1008 (size=32) 2024-11-24T22:14:31,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34013 is added to blk_1073741832_1008 (size=32) 2024-11-24T22:14:31,077 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:31,077 DEBUG [RS:0;bd6592c37bb2:41619 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c 2024-11-24T22:14:31,077 DEBUG [RS:0;bd6592c37bb2:41619 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36281 2024-11-24T22:14:31,078 DEBUG [RS:0;bd6592c37bb2:41619 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T22:14:31,081 DEBUG [RS:0;bd6592c37bb2:41619 {}] zookeeper.ZKUtil(111): regionserver:41619-0x1014c0b79ef0001, quorum=127.0.0.1:56583, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/bd6592c37bb2,41619,1732486469269 2024-11-24T22:14:31,081 WARN [RS:0;bd6592c37bb2:41619 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T22:14:31,081 INFO [RS:0;bd6592c37bb2:41619 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T22:14:31,082 DEBUG [RS:0;bd6592c37bb2:41619 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/WALs/bd6592c37bb2,41619,1732486469269 2024-11-24T22:14:31,098 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [bd6592c37bb2,41619,1732486469269] 2024-11-24T22:14:31,098 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [bd6592c37bb2,42639,1732486469476] 2024-11-24T22:14:31,098 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [bd6592c37bb2,38125,1732486469405] 2024-11-24T22:14:31,099 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-24T22:14:31,106 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T22:14:31,106 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:31,109 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T22:14:31,110 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-24T22:14:31,114 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T22:14:31,114 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:31,122 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T22:14:31,123 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-24T22:14:31,129 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T22:14:31,130 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:31,131 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T22:14:31,132 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-24T22:14:31,149 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T22:14:31,149 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:31,151 INFO [RS:2;bd6592c37bb2:42639 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T22:14:31,151 INFO [RS:1;bd6592c37bb2:38125 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T22:14:31,152 INFO [RS:0;bd6592c37bb2:41619 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T22:14:31,152 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T22:14:31,153 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T22:14:31,157 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/hbase/meta/1588230740 2024-11-24T22:14:31,158 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/hbase/meta/1588230740 2024-11-24T22:14:31,163 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T22:14:31,163 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T22:14:31,164 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T22:14:31,172 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T22:14:31,186 INFO [RS:2;bd6592c37bb2:42639 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T22:14:31,190 INFO [RS:0;bd6592c37bb2:41619 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T22:14:31,191 INFO [RS:1;bd6592c37bb2:38125 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T22:14:31,195 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T22:14:31,196 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63579963, jitterRate=-0.05258472263813019}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T22:14:31,199 INFO [RS:2;bd6592c37bb2:42639 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T22:14:31,199 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732486471077Initializing all the Stores at 1732486471080 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732486471080Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732486471099 (+19 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486471099Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732486471099Cleaning up temporary data from old regions at 1732486471163 (+64 ms)Region opened successfully at 1732486471199 (+36 ms) 2024-11-24T22:14:31,199 INFO [RS:0;bd6592c37bb2:41619 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T22:14:31,200 INFO [RS:2;bd6592c37bb2:42639 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T22:14:31,201 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T22:14:31,201 INFO [RS:0;bd6592c37bb2:41619 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T22:14:31,201 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T22:14:31,201 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T22:14:31,201 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T22:14:31,201 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T22:14:31,201 INFO [RS:1;bd6592c37bb2:38125 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T22:14:31,202 INFO [RS:1;bd6592c37bb2:38125 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T22:14:31,202 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T22:14:31,203 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732486471200Disabling compacts and flushes for region at 1732486471200Disabling writes for close at 1732486471201 (+1 ms)Writing region close event to WAL at 1732486471202 (+1 ms)Closed at 1732486471202 2024-11-24T22:14:31,203 INFO [RS:1;bd6592c37bb2:38125 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T22:14:31,205 INFO [RS:0;bd6592c37bb2:41619 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T22:14:31,209 INFO [RS:2;bd6592c37bb2:42639 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T22:14:31,209 INFO [RS:1;bd6592c37bb2:38125 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T22:14:31,211 INFO [RS:2;bd6592c37bb2:42639 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T22:14:31,212 INFO [RS:2;bd6592c37bb2:42639 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T22:14:31,212 INFO [RS:1;bd6592c37bb2:38125 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T22:14:31,212 DEBUG [RS:2;bd6592c37bb2:42639 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/bd6592c37bb2:0, corePoolSize=1, maxPoolSize=1 2024-11-24T22:14:31,212 DEBUG [RS:1;bd6592c37bb2:38125 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/bd6592c37bb2:0, corePoolSize=1, maxPoolSize=1 2024-11-24T22:14:31,212 DEBUG [RS:2;bd6592c37bb2:42639 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/bd6592c37bb2:0, corePoolSize=1, maxPoolSize=1 2024-11-24T22:14:31,212 DEBUG [RS:1;bd6592c37bb2:38125 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/bd6592c37bb2:0, corePoolSize=1, maxPoolSize=1 2024-11-24T22:14:31,212 DEBUG [RS:2;bd6592c37bb2:42639 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/bd6592c37bb2:0, corePoolSize=1, maxPoolSize=1 2024-11-24T22:14:31,212 DEBUG [RS:1;bd6592c37bb2:38125 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/bd6592c37bb2:0, corePoolSize=1, maxPoolSize=1 2024-11-24T22:14:31,212 DEBUG [RS:2;bd6592c37bb2:42639 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/bd6592c37bb2:0, corePoolSize=1, maxPoolSize=1 2024-11-24T22:14:31,212 DEBUG [RS:1;bd6592c37bb2:38125 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/bd6592c37bb2:0, corePoolSize=1, maxPoolSize=1 2024-11-24T22:14:31,213 DEBUG [RS:2;bd6592c37bb2:42639 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/bd6592c37bb2:0, corePoolSize=1, maxPoolSize=1 2024-11-24T22:14:31,213 DEBUG [RS:1;bd6592c37bb2:38125 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/bd6592c37bb2:0, corePoolSize=1, maxPoolSize=1 2024-11-24T22:14:31,213 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T22:14:31,213 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-24T22:14:31,213 DEBUG [RS:2;bd6592c37bb2:42639 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/bd6592c37bb2:0, corePoolSize=2, maxPoolSize=2 2024-11-24T22:14:31,213 DEBUG [RS:1;bd6592c37bb2:38125 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/bd6592c37bb2:0, corePoolSize=2, maxPoolSize=2 2024-11-24T22:14:31,213 DEBUG [RS:2;bd6592c37bb2:42639 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/bd6592c37bb2:0, corePoolSize=1, maxPoolSize=1 2024-11-24T22:14:31,213 DEBUG [RS:1;bd6592c37bb2:38125 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/bd6592c37bb2:0, corePoolSize=1, maxPoolSize=1 2024-11-24T22:14:31,213 DEBUG [RS:1;bd6592c37bb2:38125 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/bd6592c37bb2:0, corePoolSize=1, maxPoolSize=1 2024-11-24T22:14:31,213 DEBUG [RS:2;bd6592c37bb2:42639 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/bd6592c37bb2:0, corePoolSize=1, maxPoolSize=1 2024-11-24T22:14:31,213 DEBUG [RS:1;bd6592c37bb2:38125 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/bd6592c37bb2:0, corePoolSize=1, maxPoolSize=1 2024-11-24T22:14:31,213 DEBUG [RS:2;bd6592c37bb2:42639 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/bd6592c37bb2:0, corePoolSize=1, maxPoolSize=1 2024-11-24T22:14:31,213 DEBUG [RS:1;bd6592c37bb2:38125 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/bd6592c37bb2:0, corePoolSize=1, maxPoolSize=1 2024-11-24T22:14:31,213 DEBUG [RS:2;bd6592c37bb2:42639 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/bd6592c37bb2:0, corePoolSize=1, maxPoolSize=1 2024-11-24T22:14:31,214 DEBUG [RS:1;bd6592c37bb2:38125 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/bd6592c37bb2:0, corePoolSize=1, maxPoolSize=1 2024-11-24T22:14:31,214 DEBUG [RS:2;bd6592c37bb2:42639 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/bd6592c37bb2:0, corePoolSize=1, maxPoolSize=1 2024-11-24T22:14:31,214 DEBUG [RS:2;bd6592c37bb2:42639 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/bd6592c37bb2:0, corePoolSize=1, maxPoolSize=1 2024-11-24T22:14:31,214 INFO [RS:0;bd6592c37bb2:41619 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T22:14:31,214 DEBUG [RS:2;bd6592c37bb2:42639 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/bd6592c37bb2:0, corePoolSize=3, maxPoolSize=3 2024-11-24T22:14:31,214 INFO [RS:0;bd6592c37bb2:41619 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T22:14:31,222 DEBUG [RS:0;bd6592c37bb2:41619 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/bd6592c37bb2:0, corePoolSize=1, maxPoolSize=1 2024-11-24T22:14:31,222 DEBUG [RS:0;bd6592c37bb2:41619 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/bd6592c37bb2:0, corePoolSize=1, maxPoolSize=1 2024-11-24T22:14:31,223 DEBUG [RS:0;bd6592c37bb2:41619 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/bd6592c37bb2:0, corePoolSize=1, maxPoolSize=1 2024-11-24T22:14:31,223 DEBUG [RS:0;bd6592c37bb2:41619 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/bd6592c37bb2:0, corePoolSize=1, maxPoolSize=1 2024-11-24T22:14:31,223 DEBUG [RS:0;bd6592c37bb2:41619 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/bd6592c37bb2:0, corePoolSize=1, maxPoolSize=1 2024-11-24T22:14:31,223 DEBUG [RS:0;bd6592c37bb2:41619 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/bd6592c37bb2:0, corePoolSize=2, maxPoolSize=2 2024-11-24T22:14:31,223 DEBUG [RS:0;bd6592c37bb2:41619 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/bd6592c37bb2:0, corePoolSize=1, maxPoolSize=1 2024-11-24T22:14:31,223 DEBUG [RS:0;bd6592c37bb2:41619 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/bd6592c37bb2:0, corePoolSize=1, maxPoolSize=1 2024-11-24T22:14:31,224 DEBUG [RS:0;bd6592c37bb2:41619 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/bd6592c37bb2:0, corePoolSize=1, maxPoolSize=1 2024-11-24T22:14:31,224 DEBUG [RS:0;bd6592c37bb2:41619 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/bd6592c37bb2:0, corePoolSize=1, maxPoolSize=1 2024-11-24T22:14:31,224 DEBUG [RS:0;bd6592c37bb2:41619 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/bd6592c37bb2:0, corePoolSize=1, maxPoolSize=1 2024-11-24T22:14:31,225 DEBUG [RS:0;bd6592c37bb2:41619 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/bd6592c37bb2:0, corePoolSize=1, maxPoolSize=1 2024-11-24T22:14:31,225 DEBUG [RS:0;bd6592c37bb2:41619 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/bd6592c37bb2:0, corePoolSize=3, maxPoolSize=3 2024-11-24T22:14:31,225 DEBUG [RS:0;bd6592c37bb2:41619 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0, corePoolSize=3, maxPoolSize=3 2024-11-24T22:14:31,214 DEBUG [RS:2;bd6592c37bb2:42639 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0, corePoolSize=3, maxPoolSize=3 2024-11-24T22:14:31,228 INFO [RS:0;bd6592c37bb2:41619 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T22:14:31,228 INFO [RS:0;bd6592c37bb2:41619 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T22:14:31,229 INFO [RS:0;bd6592c37bb2:41619 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T22:14:31,229 INFO [RS:0;bd6592c37bb2:41619 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T22:14:31,229 INFO [RS:0;bd6592c37bb2:41619 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T22:14:31,229 INFO [RS:0;bd6592c37bb2:41619 {}] hbase.ChoreService(168): Chore ScheduledChore name=bd6592c37bb2,41619,1732486469269-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T22:14:31,217 DEBUG [RS:1;bd6592c37bb2:38125 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/bd6592c37bb2:0, corePoolSize=1, maxPoolSize=1 2024-11-24T22:14:31,236 DEBUG [RS:1;bd6592c37bb2:38125 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/bd6592c37bb2:0, corePoolSize=3, maxPoolSize=3 2024-11-24T22:14:31,236 DEBUG [RS:1;bd6592c37bb2:38125 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/bd6592c37bb2:0, corePoolSize=3, maxPoolSize=3 2024-11-24T22:14:31,241 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-24T22:14:31,259 INFO [RS:2;bd6592c37bb2:42639 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T22:14:31,260 INFO [RS:2;bd6592c37bb2:42639 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T22:14:31,260 INFO [RS:2;bd6592c37bb2:42639 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T22:14:31,260 INFO [RS:2;bd6592c37bb2:42639 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T22:14:31,260 INFO [RS:2;bd6592c37bb2:42639 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T22:14:31,260 INFO [RS:2;bd6592c37bb2:42639 {}] hbase.ChoreService(168): Chore ScheduledChore name=bd6592c37bb2,42639,1732486469476-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T22:14:31,277 INFO [RS:0;bd6592c37bb2:41619 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T22:14:31,279 INFO [RS:0;bd6592c37bb2:41619 {}] hbase.ChoreService(168): Chore ScheduledChore name=bd6592c37bb2,41619,1732486469269-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T22:14:31,279 INFO [RS:0;bd6592c37bb2:41619 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T22:14:31,280 INFO [RS:0;bd6592c37bb2:41619 {}] regionserver.Replication(171): bd6592c37bb2,41619,1732486469269 started 2024-11-24T22:14:31,281 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T22:14:31,286 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-24T22:14:31,291 INFO [RS:2;bd6592c37bb2:42639 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T22:14:31,291 INFO [RS:2;bd6592c37bb2:42639 {}] hbase.ChoreService(168): Chore ScheduledChore name=bd6592c37bb2,42639,1732486469476-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T22:14:31,292 INFO [RS:2;bd6592c37bb2:42639 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T22:14:31,292 INFO [RS:2;bd6592c37bb2:42639 {}] regionserver.Replication(171): bd6592c37bb2,42639,1732486469476 started 2024-11-24T22:14:31,294 INFO [RS:1;bd6592c37bb2:38125 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T22:14:31,294 INFO [RS:1;bd6592c37bb2:38125 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T22:14:31,294 INFO [RS:1;bd6592c37bb2:38125 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T22:14:31,295 INFO [RS:1;bd6592c37bb2:38125 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T22:14:31,295 INFO [RS:1;bd6592c37bb2:38125 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T22:14:31,295 INFO [RS:1;bd6592c37bb2:38125 {}] hbase.ChoreService(168): Chore ScheduledChore name=bd6592c37bb2,38125,1732486469405-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T22:14:31,328 INFO [RS:2;bd6592c37bb2:42639 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T22:14:31,329 INFO [RS:2;bd6592c37bb2:42639 {}] regionserver.HRegionServer(1482): Serving as bd6592c37bb2,42639,1732486469476, RpcServer on bd6592c37bb2/172.17.0.2:42639, sessionid=0x1014c0b79ef0003 2024-11-24T22:14:31,329 DEBUG [RS:2;bd6592c37bb2:42639 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T22:14:31,330 DEBUG [RS:2;bd6592c37bb2:42639 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:31,330 DEBUG [RS:2;bd6592c37bb2:42639 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'bd6592c37bb2,42639,1732486469476' 2024-11-24T22:14:31,330 DEBUG [RS:2;bd6592c37bb2:42639 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T22:14:31,331 DEBUG [RS:2;bd6592c37bb2:42639 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T22:14:31,332 DEBUG [RS:2;bd6592c37bb2:42639 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T22:14:31,332 DEBUG [RS:2;bd6592c37bb2:42639 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T22:14:31,333 DEBUG [RS:2;bd6592c37bb2:42639 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:31,333 DEBUG [RS:2;bd6592c37bb2:42639 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'bd6592c37bb2,42639,1732486469476' 2024-11-24T22:14:31,333 DEBUG [RS:2;bd6592c37bb2:42639 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T22:14:31,333 INFO [RS:0;bd6592c37bb2:41619 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T22:14:31,333 INFO [RS:0;bd6592c37bb2:41619 {}] regionserver.HRegionServer(1482): Serving as bd6592c37bb2,41619,1732486469269, RpcServer on bd6592c37bb2/172.17.0.2:41619, sessionid=0x1014c0b79ef0001 2024-11-24T22:14:31,333 DEBUG [RS:0;bd6592c37bb2:41619 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T22:14:31,333 DEBUG [RS:0;bd6592c37bb2:41619 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager bd6592c37bb2,41619,1732486469269 2024-11-24T22:14:31,333 DEBUG [RS:0;bd6592c37bb2:41619 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'bd6592c37bb2,41619,1732486469269' 2024-11-24T22:14:31,334 DEBUG [RS:0;bd6592c37bb2:41619 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T22:14:31,334 DEBUG [RS:2;bd6592c37bb2:42639 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T22:14:31,335 DEBUG [RS:2;bd6592c37bb2:42639 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T22:14:31,335 INFO [RS:2;bd6592c37bb2:42639 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T22:14:31,335 INFO [RS:2;bd6592c37bb2:42639 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T22:14:31,336 INFO [RS:1;bd6592c37bb2:38125 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T22:14:31,336 INFO [RS:1;bd6592c37bb2:38125 {}] hbase.ChoreService(168): Chore ScheduledChore name=bd6592c37bb2,38125,1732486469405-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T22:14:31,336 INFO [RS:1;bd6592c37bb2:38125 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T22:14:31,336 DEBUG [RS:0;bd6592c37bb2:41619 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T22:14:31,336 INFO [RS:1;bd6592c37bb2:38125 {}] regionserver.Replication(171): bd6592c37bb2,38125,1732486469405 started 2024-11-24T22:14:31,337 DEBUG [RS:0;bd6592c37bb2:41619 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T22:14:31,337 DEBUG [RS:0;bd6592c37bb2:41619 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T22:14:31,337 DEBUG [RS:0;bd6592c37bb2:41619 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager bd6592c37bb2,41619,1732486469269 2024-11-24T22:14:31,337 DEBUG [RS:0;bd6592c37bb2:41619 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'bd6592c37bb2,41619,1732486469269' 2024-11-24T22:14:31,337 DEBUG [RS:0;bd6592c37bb2:41619 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T22:14:31,338 DEBUG [RS:0;bd6592c37bb2:41619 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T22:14:31,339 DEBUG [RS:0;bd6592c37bb2:41619 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T22:14:31,339 INFO [RS:0;bd6592c37bb2:41619 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T22:14:31,339 INFO [RS:0;bd6592c37bb2:41619 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T22:14:31,356 INFO [RS:1;bd6592c37bb2:38125 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T22:14:31,357 INFO [RS:1;bd6592c37bb2:38125 {}] regionserver.HRegionServer(1482): Serving as bd6592c37bb2,38125,1732486469405, RpcServer on bd6592c37bb2/172.17.0.2:38125, sessionid=0x1014c0b79ef0002 2024-11-24T22:14:31,357 DEBUG [RS:1;bd6592c37bb2:38125 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T22:14:31,357 DEBUG [RS:1;bd6592c37bb2:38125 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:31,357 DEBUG [RS:1;bd6592c37bb2:38125 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'bd6592c37bb2,38125,1732486469405' 2024-11-24T22:14:31,357 DEBUG [RS:1;bd6592c37bb2:38125 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T22:14:31,358 DEBUG [RS:1;bd6592c37bb2:38125 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T22:14:31,359 DEBUG [RS:1;bd6592c37bb2:38125 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T22:14:31,359 DEBUG [RS:1;bd6592c37bb2:38125 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T22:14:31,359 DEBUG [RS:1;bd6592c37bb2:38125 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:31,359 DEBUG [RS:1;bd6592c37bb2:38125 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'bd6592c37bb2,38125,1732486469405' 2024-11-24T22:14:31,359 DEBUG [RS:1;bd6592c37bb2:38125 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T22:14:31,360 DEBUG [RS:1;bd6592c37bb2:38125 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T22:14:31,361 DEBUG [RS:1;bd6592c37bb2:38125 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T22:14:31,361 INFO [RS:1;bd6592c37bb2:38125 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T22:14:31,361 INFO [RS:1;bd6592c37bb2:38125 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T22:14:31,437 WARN [bd6592c37bb2:36279 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T22:14:31,442 INFO [RS:2;bd6592c37bb2:42639 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T22:14:31,442 INFO [RS:0;bd6592c37bb2:41619 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T22:14:31,446 INFO [RS:2;bd6592c37bb2:42639 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=bd6592c37bb2%2C42639%2C1732486469476, suffix=, logDir=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/WALs/bd6592c37bb2,42639,1732486469476, archiveDir=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/oldWALs, maxLogs=32 2024-11-24T22:14:31,447 INFO [RS:0;bd6592c37bb2:41619 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=bd6592c37bb2%2C41619%2C1732486469269, suffix=, logDir=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/WALs/bd6592c37bb2,41619,1732486469269, archiveDir=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/oldWALs, maxLogs=32 2024-11-24T22:14:31,462 INFO [RS:1;bd6592c37bb2:38125 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T22:14:31,465 INFO [RS:1;bd6592c37bb2:38125 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=bd6592c37bb2%2C38125%2C1732486469405, suffix=, logDir=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/WALs/bd6592c37bb2,38125,1732486469405, archiveDir=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/oldWALs, maxLogs=32 2024-11-24T22:14:31,467 DEBUG [RS:2;bd6592c37bb2:42639 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/WALs/bd6592c37bb2,42639,1732486469476/bd6592c37bb2%2C42639%2C1732486469476.1732486471450, exclude list is [], retry=0 2024-11-24T22:14:31,472 DEBUG [RS:0;bd6592c37bb2:41619 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/WALs/bd6592c37bb2,41619,1732486469269/bd6592c37bb2%2C41619%2C1732486469269.1732486471450, exclude list is [], retry=0 2024-11-24T22:14:31,479 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35383,DS-56b705de-da8a-4837-9f73-0e8d9391298c,DISK] 2024-11-24T22:14:31,480 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34899,DS-fb965c9f-83ab-43e9-b4f1-51c95ecaf7db,DISK] 2024-11-24T22:14:31,481 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34013,DS-e98687a1-55aa-42d3-a359-80f43904e747,DISK] 2024-11-24T22:14:31,482 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34899,DS-fb965c9f-83ab-43e9-b4f1-51c95ecaf7db,DISK] 2024-11-24T22:14:31,482 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35383,DS-56b705de-da8a-4837-9f73-0e8d9391298c,DISK] 2024-11-24T22:14:31,483 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34013,DS-e98687a1-55aa-42d3-a359-80f43904e747,DISK] 2024-11-24T22:14:31,488 DEBUG [RS:1;bd6592c37bb2:38125 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/WALs/bd6592c37bb2,38125,1732486469405/bd6592c37bb2%2C38125%2C1732486469405.1732486471468, exclude list is [], retry=0 2024-11-24T22:14:31,495 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34899,DS-fb965c9f-83ab-43e9-b4f1-51c95ecaf7db,DISK] 2024-11-24T22:14:31,495 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35383,DS-56b705de-da8a-4837-9f73-0e8d9391298c,DISK] 2024-11-24T22:14:31,495 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34013,DS-e98687a1-55aa-42d3-a359-80f43904e747,DISK] 2024-11-24T22:14:31,513 INFO [RS:2;bd6592c37bb2:42639 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/WALs/bd6592c37bb2,42639,1732486469476/bd6592c37bb2%2C42639%2C1732486469476.1732486471450 2024-11-24T22:14:31,520 INFO [RS:0;bd6592c37bb2:41619 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/WALs/bd6592c37bb2,41619,1732486469269/bd6592c37bb2%2C41619%2C1732486469269.1732486471450 2024-11-24T22:14:31,524 DEBUG [RS:2;bd6592c37bb2:42639 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38755:38755),(127.0.0.1/127.0.0.1:42065:42065),(127.0.0.1/127.0.0.1:36251:36251)] 2024-11-24T22:14:31,524 DEBUG [RS:0;bd6592c37bb2:41619 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38755:38755),(127.0.0.1/127.0.0.1:42065:42065),(127.0.0.1/127.0.0.1:36251:36251)] 2024-11-24T22:14:31,527 INFO [RS:1;bd6592c37bb2:38125 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/WALs/bd6592c37bb2,38125,1732486469405/bd6592c37bb2%2C38125%2C1732486469405.1732486471468 2024-11-24T22:14:31,528 DEBUG [RS:1;bd6592c37bb2:38125 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36251:36251),(127.0.0.1/127.0.0.1:42065:42065),(127.0.0.1/127.0.0.1:38755:38755)] 2024-11-24T22:14:31,691 DEBUG [bd6592c37bb2:36279 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-24T22:14:31,701 DEBUG [bd6592c37bb2:36279 {}] balancer.BalancerClusterState(204): Hosts are {bd6592c37bb2=0} racks are {/default-rack=0} 2024-11-24T22:14:31,709 DEBUG [bd6592c37bb2:36279 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T22:14:31,710 DEBUG [bd6592c37bb2:36279 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T22:14:31,710 DEBUG [bd6592c37bb2:36279 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-24T22:14:31,710 DEBUG [bd6592c37bb2:36279 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T22:14:31,710 DEBUG [bd6592c37bb2:36279 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T22:14:31,710 DEBUG [bd6592c37bb2:36279 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-24T22:14:31,710 INFO [bd6592c37bb2:36279 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T22:14:31,710 INFO [bd6592c37bb2:36279 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T22:14:31,710 INFO [bd6592c37bb2:36279 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-24T22:14:31,710 DEBUG [bd6592c37bb2:36279 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T22:14:31,719 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=bd6592c37bb2,41619,1732486469269 2024-11-24T22:14:31,726 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bd6592c37bb2,41619,1732486469269, state=OPENING 2024-11-24T22:14:31,732 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-24T22:14:31,735 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36279-0x1014c0b79ef0000, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T22:14:31,735 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41619-0x1014c0b79ef0001, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T22:14:31,735 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42639-0x1014c0b79ef0003, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T22:14:31,735 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38125-0x1014c0b79ef0002, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T22:14:31,736 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:31,736 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:31,737 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:31,737 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:31,739 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T22:14:31,741 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=bd6592c37bb2,41619,1732486469269}] 2024-11-24T22:14:31,922 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T22:14:31,925 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52845, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T22:14:31,938 INFO [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-24T22:14:31,939 INFO [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T22:14:31,939 INFO [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-24T22:14:31,943 INFO [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=bd6592c37bb2%2C41619%2C1732486469269.meta, suffix=.meta, logDir=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/WALs/bd6592c37bb2,41619,1732486469269, archiveDir=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/oldWALs, maxLogs=32 2024-11-24T22:14:31,962 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/WALs/bd6592c37bb2,41619,1732486469269/bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, exclude list is [], retry=0 2024-11-24T22:14:31,967 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34899,DS-fb965c9f-83ab-43e9-b4f1-51c95ecaf7db,DISK] 2024-11-24T22:14:31,967 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34013,DS-e98687a1-55aa-42d3-a359-80f43904e747,DISK] 2024-11-24T22:14:31,972 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35383,DS-56b705de-da8a-4837-9f73-0e8d9391298c,DISK] 2024-11-24T22:14:31,979 INFO [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/WALs/bd6592c37bb2,41619,1732486469269/bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta 2024-11-24T22:14:31,980 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42065:42065),(127.0.0.1/127.0.0.1:36251:36251),(127.0.0.1/127.0.0.1:38755:38755)] 2024-11-24T22:14:31,981 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-24T22:14:31,983 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-24T22:14:31,987 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-24T22:14:31,993 INFO [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-24T22:14:31,999 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-24T22:14:32,000 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:32,000 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-24T22:14:32,000 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-24T22:14:32,004 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-24T22:14:32,006 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T22:14:32,006 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:32,007 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T22:14:32,007 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-24T22:14:32,009 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T22:14:32,009 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:32,010 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T22:14:32,010 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-24T22:14:32,011 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T22:14:32,012 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:32,012 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T22:14:32,013 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-24T22:14:32,014 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T22:14:32,015 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:32,016 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T22:14:32,016 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T22:14:32,018 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/hbase/meta/1588230740 2024-11-24T22:14:32,021 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/hbase/meta/1588230740 2024-11-24T22:14:32,026 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T22:14:32,026 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T22:14:32,027 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T22:14:32,030 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T22:14:32,033 INFO [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68593711, jitterRate=0.02212594449520111}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T22:14:32,034 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-24T22:14:32,036 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732486472001Writing region info on filesystem at 1732486472001Initializing all the Stores at 1732486472003 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732486472003Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732486472004 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486472004Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732486472004Cleaning up temporary data from old regions at 1732486472026 (+22 ms)Running coprocessor post-open hooks at 1732486472034 (+8 ms)Region opened successfully at 1732486472036 (+2 ms) 2024-11-24T22:14:32,048 INFO [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732486471912 2024-11-24T22:14:32,071 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=bd6592c37bb2,41619,1732486469269 2024-11-24T22:14:32,071 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-24T22:14:32,072 INFO [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-24T22:14:32,075 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bd6592c37bb2,41619,1732486469269, state=OPEN 2024-11-24T22:14:32,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42639-0x1014c0b79ef0003, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:32,078 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:32,079 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36279-0x1014c0b79ef0000, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:32,080 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38125-0x1014c0b79ef0002, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:32,080 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:32,080 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:32,080 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41619-0x1014c0b79ef0001, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:32,080 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:32,082 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=bd6592c37bb2,41619,1732486469269 2024-11-24T22:14:32,090 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-24T22:14:32,090 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bd6592c37bb2,41619,1732486469269 in 341 msec 2024-11-24T22:14:32,101 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-24T22:14:32,101 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 857 msec 2024-11-24T22:14:32,104 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T22:14:32,104 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-24T22:14:32,133 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:14:32,135 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,41619,1732486469269, seqNum=-1] 2024-11-24T22:14:32,163 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:14:32,166 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56235, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:14:32,198 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.5040 sec 2024-11-24T22:14:32,199 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732486472199, completionTime=-1 2024-11-24T22:14:32,203 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-24T22:14:32,203 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T22:14:32,233 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-24T22:14:32,233 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732486532233 2024-11-24T22:14:32,234 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732486592233 2024-11-24T22:14:32,234 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 30 msec 2024-11-24T22:14:32,235 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-24T22:14:32,243 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bd6592c37bb2,36279,1732486468255-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T22:14:32,244 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bd6592c37bb2,36279,1732486468255-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T22:14:32,244 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bd6592c37bb2,36279,1732486468255-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T22:14:32,246 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-bd6592c37bb2:36279, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T22:14:32,247 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-24T22:14:32,252 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-24T22:14:32,257 DEBUG [master/bd6592c37bb2:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-24T22:14:32,293 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.697sec 2024-11-24T22:14:32,296 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-24T22:14:32,298 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-24T22:14:32,299 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-24T22:14:32,300 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-24T22:14:32,300 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-24T22:14:32,301 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bd6592c37bb2,36279,1732486468255-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T22:14:32,302 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bd6592c37bb2,36279,1732486468255-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-24T22:14:32,313 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-24T22:14:32,314 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-24T22:14:32,315 INFO [master/bd6592c37bb2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bd6592c37bb2,36279,1732486468255-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T22:14:32,333 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@664a870e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:32,339 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-24T22:14:32,340 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-24T22:14:32,345 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bd6592c37bb2,36279,-1 for getting cluster id 2024-11-24T22:14:32,349 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T22:14:32,360 DEBUG [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '905a4a1c-dd04-42c5-bf3f-b8b8bda6274a' 2024-11-24T22:14:32,363 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T22:14:32,364 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "905a4a1c-dd04-42c5-bf3f-b8b8bda6274a" 2024-11-24T22:14:32,364 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@566db3d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:32,364 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bd6592c37bb2,36279,-1] 2024-11-24T22:14:32,368 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T22:14:32,374 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:32,378 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56236, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T22:14:32,381 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62ac722f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T22:14:32,382 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:14:32,391 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,41619,1732486469269, seqNum=-1] 2024-11-24T22:14:32,391 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:14:32,394 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36236, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:14:32,424 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=bd6592c37bb2,36279,1732486468255 2024-11-24T22:14:32,429 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T22:14:32,475 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is bd6592c37bb2,36279,1732486468255 2024-11-24T22:14:32,478 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@602ac356 2024-11-24T22:14:32,479 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T22:14:32,484 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56238, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T22:14:32,494 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36279 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'test', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T22:14:32,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36279 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test 2024-11-24T22:14:32,522 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T22:14:32,526 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:32,529 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T22:14:32,533 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36279 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "test" procId is: 4 2024-11-24T22:14:32,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36279 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T22:14:32,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741837_1013 (size=358) 2024-11-24T22:14:32,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34013 is added to blk_1073741837_1013 (size=358) 2024-11-24T22:14:32,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35383 is added to blk_1073741837_1013 (size=358) 2024-11-24T22:14:32,581 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4e91326d9778777879bc895ecb241af8, NAME => 'test,,1732486472486.4e91326d9778777879bc895ecb241af8.', STARTKEY => '', ENDKEY => 'bbb'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c 2024-11-24T22:14:32,589 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 4279880609a62ccb9e9fb29e79bdf7e7, NAME => 'test,bbb,1732486472486.4279880609a62ccb9e9fb29e79bdf7e7.', STARTKEY => 'bbb', ENDKEY => 'ccc'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c 2024-11-24T22:14:32,591 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => c68a44a11eeaccf8a3c2f49ba1446c1e, NAME => 'test,ccc,1732486472486.c68a44a11eeaccf8a3c2f49ba1446c1e.', STARTKEY => 'ccc', ENDKEY => 'ddd'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c 2024-11-24T22:14:32,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741838_1014 (size=42) 2024-11-24T22:14:32,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34013 is added to blk_1073741838_1014 (size=42) 2024-11-24T22:14:32,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35383 is added to blk_1073741838_1014 (size=42) 2024-11-24T22:14:32,634 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,,1732486472486.4e91326d9778777879bc895ecb241af8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:32,634 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 4e91326d9778777879bc895ecb241af8, disabling compactions & flushes 2024-11-24T22:14:32,635 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,,1732486472486.4e91326d9778777879bc895ecb241af8. 2024-11-24T22:14:32,635 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1732486472486.4e91326d9778777879bc895ecb241af8. 2024-11-24T22:14:32,635 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,,1732486472486.4e91326d9778777879bc895ecb241af8. after waiting 0 ms 2024-11-24T22:14:32,635 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,,1732486472486.4e91326d9778777879bc895ecb241af8. 2024-11-24T22:14:32,635 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,,1732486472486.4e91326d9778777879bc895ecb241af8. 2024-11-24T22:14:32,635 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4e91326d9778777879bc895ecb241af8: Waiting for close lock at 1732486472634Disabling compacts and flushes for region at 1732486472634Disabling writes for close at 1732486472635 (+1 ms)Writing region close event to WAL at 1732486472635Closed at 1732486472635 2024-11-24T22:14:32,636 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 698dd9d64bdaf6a81e797d2590ff86f9, NAME => 'test,ddd,1732486472486.698dd9d64bdaf6a81e797d2590ff86f9.', STARTKEY => 'ddd', ENDKEY => 'eee'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c 2024-11-24T22:14:32,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36279 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T22:14:32,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35383 is added to blk_1073741839_1015 (size=45) 2024-11-24T22:14:32,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34013 is added to blk_1073741839_1015 (size=45) 2024-11-24T22:14:32,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741839_1015 (size=45) 2024-11-24T22:14:32,669 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ccc,1732486472486.c68a44a11eeaccf8a3c2f49ba1446c1e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:32,669 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing c68a44a11eeaccf8a3c2f49ba1446c1e, disabling compactions & flushes 2024-11-24T22:14:32,669 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ccc,1732486472486.c68a44a11eeaccf8a3c2f49ba1446c1e. 2024-11-24T22:14:32,669 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1732486472486.c68a44a11eeaccf8a3c2f49ba1446c1e. 2024-11-24T22:14:32,669 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1732486472486.c68a44a11eeaccf8a3c2f49ba1446c1e. after waiting 0 ms 2024-11-24T22:14:32,669 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1732486472486.c68a44a11eeaccf8a3c2f49ba1446c1e. 2024-11-24T22:14:32,670 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ccc,1732486472486.c68a44a11eeaccf8a3c2f49ba1446c1e. 2024-11-24T22:14:32,670 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for c68a44a11eeaccf8a3c2f49ba1446c1e: Waiting for close lock at 1732486472669Disabling compacts and flushes for region at 1732486472669Disabling writes for close at 1732486472669Writing region close event to WAL at 1732486472669Closed at 1732486472669 2024-11-24T22:14:32,670 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 46b79f5a3db63445091682b5e72f635a, NAME => 'test,eee,1732486472486.46b79f5a3db63445091682b5e72f635a.', STARTKEY => 'eee', ENDKEY => 'fff'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c 2024-11-24T22:14:32,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35383 is added to blk_1073741840_1016 (size=45) 2024-11-24T22:14:32,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741840_1016 (size=45) 2024-11-24T22:14:32,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34013 is added to blk_1073741840_1016 (size=45) 2024-11-24T22:14:32,693 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,bbb,1732486472486.4279880609a62ccb9e9fb29e79bdf7e7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:32,693 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 4279880609a62ccb9e9fb29e79bdf7e7, disabling compactions & flushes 2024-11-24T22:14:32,693 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,bbb,1732486472486.4279880609a62ccb9e9fb29e79bdf7e7. 2024-11-24T22:14:32,693 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1732486472486.4279880609a62ccb9e9fb29e79bdf7e7. 2024-11-24T22:14:32,693 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1732486472486.4279880609a62ccb9e9fb29e79bdf7e7. after waiting 0 ms 2024-11-24T22:14:32,693 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1732486472486.4279880609a62ccb9e9fb29e79bdf7e7. 2024-11-24T22:14:32,693 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,bbb,1732486472486.4279880609a62ccb9e9fb29e79bdf7e7. 2024-11-24T22:14:32,693 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 4279880609a62ccb9e9fb29e79bdf7e7: Waiting for close lock at 1732486472693Disabling compacts and flushes for region at 1732486472693Disabling writes for close at 1732486472693Writing region close event to WAL at 1732486472693Closed at 1732486472693 2024-11-24T22:14:32,694 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 736f44b8f348e2a5a96dfd3abc5e62b2, NAME => 'test,fff,1732486472486.736f44b8f348e2a5a96dfd3abc5e62b2.', STARTKEY => 'fff', ENDKEY => 'ggg'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c 2024-11-24T22:14:32,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34013 is added to blk_1073741841_1017 (size=45) 2024-11-24T22:14:32,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741841_1017 (size=45) 2024-11-24T22:14:32,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35383 is added to blk_1073741841_1017 (size=45) 2024-11-24T22:14:32,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35383 is added to blk_1073741842_1018 (size=45) 2024-11-24T22:14:32,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741842_1018 (size=45) 2024-11-24T22:14:32,709 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ddd,1732486472486.698dd9d64bdaf6a81e797d2590ff86f9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:32,709 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 698dd9d64bdaf6a81e797d2590ff86f9, disabling compactions & flushes 2024-11-24T22:14:32,709 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ddd,1732486472486.698dd9d64bdaf6a81e797d2590ff86f9. 2024-11-24T22:14:32,709 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1732486472486.698dd9d64bdaf6a81e797d2590ff86f9. 2024-11-24T22:14:32,709 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1732486472486.698dd9d64bdaf6a81e797d2590ff86f9. after waiting 0 ms 2024-11-24T22:14:32,709 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1732486472486.698dd9d64bdaf6a81e797d2590ff86f9. 2024-11-24T22:14:32,709 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ddd,1732486472486.698dd9d64bdaf6a81e797d2590ff86f9. 2024-11-24T22:14:32,709 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 698dd9d64bdaf6a81e797d2590ff86f9: Waiting for close lock at 1732486472709Disabling compacts and flushes for region at 1732486472709Disabling writes for close at 1732486472709Writing region close event to WAL at 1732486472709Closed at 1732486472709 2024-11-24T22:14:32,710 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f72c935213180e3ff989af795022b7e4, NAME => 'test,ggg,1732486472486.f72c935213180e3ff989af795022b7e4.', STARTKEY => 'ggg', ENDKEY => 'hhh'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c 2024-11-24T22:14:32,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34013 is added to blk_1073741842_1018 (size=45) 2024-11-24T22:14:32,718 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,eee,1732486472486.46b79f5a3db63445091682b5e72f635a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:32,718 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 46b79f5a3db63445091682b5e72f635a, disabling compactions & flushes 2024-11-24T22:14:32,719 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,eee,1732486472486.46b79f5a3db63445091682b5e72f635a. 2024-11-24T22:14:32,719 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1732486472486.46b79f5a3db63445091682b5e72f635a. 2024-11-24T22:14:32,719 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,eee,1732486472486.46b79f5a3db63445091682b5e72f635a. after waiting 0 ms 2024-11-24T22:14:32,719 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,eee,1732486472486.46b79f5a3db63445091682b5e72f635a. 2024-11-24T22:14:32,719 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,eee,1732486472486.46b79f5a3db63445091682b5e72f635a. 2024-11-24T22:14:32,719 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 46b79f5a3db63445091682b5e72f635a: Waiting for close lock at 1732486472718Disabling compacts and flushes for region at 1732486472718Disabling writes for close at 1732486472719 (+1 ms)Writing region close event to WAL at 1732486472719Closed at 1732486472719 2024-11-24T22:14:32,720 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 72a41015f7b88194cc3a08feb66aa4f9, NAME => 'test,hhh,1732486472486.72a41015f7b88194cc3a08feb66aa4f9.', STARTKEY => 'hhh', ENDKEY => 'iii'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c 2024-11-24T22:14:32,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34013 is added to blk_1073741843_1019 (size=45) 2024-11-24T22:14:32,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741843_1019 (size=45) 2024-11-24T22:14:32,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35383 is added to blk_1073741843_1019 (size=45) 2024-11-24T22:14:32,753 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,fff,1732486472486.736f44b8f348e2a5a96dfd3abc5e62b2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:32,756 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 736f44b8f348e2a5a96dfd3abc5e62b2, disabling compactions & flushes 2024-11-24T22:14:32,756 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,fff,1732486472486.736f44b8f348e2a5a96dfd3abc5e62b2. 2024-11-24T22:14:32,756 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1732486472486.736f44b8f348e2a5a96dfd3abc5e62b2. 2024-11-24T22:14:32,757 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,fff,1732486472486.736f44b8f348e2a5a96dfd3abc5e62b2. after waiting 0 ms 2024-11-24T22:14:32,757 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,fff,1732486472486.736f44b8f348e2a5a96dfd3abc5e62b2. 2024-11-24T22:14:32,757 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,fff,1732486472486.736f44b8f348e2a5a96dfd3abc5e62b2. 2024-11-24T22:14:32,757 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 736f44b8f348e2a5a96dfd3abc5e62b2: Waiting for close lock at 1732486472756Disabling compacts and flushes for region at 1732486472756Disabling writes for close at 1732486472757 (+1 ms)Writing region close event to WAL at 1732486472757Closed at 1732486472757 2024-11-24T22:14:32,758 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 94f2fa48145fe3bdeffbfded6a4edd73, NAME => 'test,iii,1732486472486.94f2fa48145fe3bdeffbfded6a4edd73.', STARTKEY => 'iii', ENDKEY => 'jjj'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c 2024-11-24T22:14:32,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741844_1020 (size=45) 2024-11-24T22:14:32,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34013 is added to blk_1073741844_1020 (size=45) 2024-11-24T22:14:32,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35383 is added to blk_1073741844_1020 (size=45) 2024-11-24T22:14:32,765 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ggg,1732486472486.f72c935213180e3ff989af795022b7e4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:32,765 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing f72c935213180e3ff989af795022b7e4, disabling compactions & flushes 2024-11-24T22:14:32,765 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ggg,1732486472486.f72c935213180e3ff989af795022b7e4. 2024-11-24T22:14:32,765 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1732486472486.f72c935213180e3ff989af795022b7e4. 2024-11-24T22:14:32,766 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1732486472486.f72c935213180e3ff989af795022b7e4. after waiting 0 ms 2024-11-24T22:14:32,766 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1732486472486.f72c935213180e3ff989af795022b7e4. 2024-11-24T22:14:32,766 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ggg,1732486472486.f72c935213180e3ff989af795022b7e4. 2024-11-24T22:14:32,766 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for f72c935213180e3ff989af795022b7e4: Waiting for close lock at 1732486472765Disabling compacts and flushes for region at 1732486472765Disabling writes for close at 1732486472766 (+1 ms)Writing region close event to WAL at 1732486472766Closed at 1732486472766 2024-11-24T22:14:32,766 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9b2e9ce57ffd0cd61c72602e5f6d9cae, NAME => 'test,jjj,1732486472486.9b2e9ce57ffd0cd61c72602e5f6d9cae.', STARTKEY => 'jjj', ENDKEY => 'kkk'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c 2024-11-24T22:14:32,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741845_1021 (size=45) 2024-11-24T22:14:32,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34013 is added to blk_1073741845_1021 (size=45) 2024-11-24T22:14:32,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35383 is added to blk_1073741845_1021 (size=45) 2024-11-24T22:14:32,786 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,hhh,1732486472486.72a41015f7b88194cc3a08feb66aa4f9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:32,786 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 72a41015f7b88194cc3a08feb66aa4f9, disabling compactions & flushes 2024-11-24T22:14:32,786 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,hhh,1732486472486.72a41015f7b88194cc3a08feb66aa4f9. 2024-11-24T22:14:32,786 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1732486472486.72a41015f7b88194cc3a08feb66aa4f9. 2024-11-24T22:14:32,786 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1732486472486.72a41015f7b88194cc3a08feb66aa4f9. after waiting 0 ms 2024-11-24T22:14:32,786 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1732486472486.72a41015f7b88194cc3a08feb66aa4f9. 2024-11-24T22:14:32,786 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,hhh,1732486472486.72a41015f7b88194cc3a08feb66aa4f9. 2024-11-24T22:14:32,786 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 72a41015f7b88194cc3a08feb66aa4f9: Waiting for close lock at 1732486472786Disabling compacts and flushes for region at 1732486472786Disabling writes for close at 1732486472786Writing region close event to WAL at 1732486472786Closed at 1732486472786 2024-11-24T22:14:32,787 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 0b86957c7b81466a818bbff45fac751f, NAME => 'test,kkk,1732486472486.0b86957c7b81466a818bbff45fac751f.', STARTKEY => 'kkk', ENDKEY => 'lll'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c 2024-11-24T22:14:32,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741847_1023 (size=45) 2024-11-24T22:14:32,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35383 is added to blk_1073741847_1023 (size=45) 2024-11-24T22:14:32,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34013 is added to blk_1073741847_1023 (size=45) 2024-11-24T22:14:32,811 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,iii,1732486472486.94f2fa48145fe3bdeffbfded6a4edd73.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:32,811 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 94f2fa48145fe3bdeffbfded6a4edd73, disabling compactions & flushes 2024-11-24T22:14:32,811 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,iii,1732486472486.94f2fa48145fe3bdeffbfded6a4edd73. 2024-11-24T22:14:32,811 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1732486472486.94f2fa48145fe3bdeffbfded6a4edd73. 2024-11-24T22:14:32,812 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,iii,1732486472486.94f2fa48145fe3bdeffbfded6a4edd73. after waiting 0 ms 2024-11-24T22:14:32,812 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,iii,1732486472486.94f2fa48145fe3bdeffbfded6a4edd73. 2024-11-24T22:14:32,812 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,iii,1732486472486.94f2fa48145fe3bdeffbfded6a4edd73. 2024-11-24T22:14:32,812 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 94f2fa48145fe3bdeffbfded6a4edd73: Waiting for close lock at 1732486472811Disabling compacts and flushes for region at 1732486472811Disabling writes for close at 1732486472812 (+1 ms)Writing region close event to WAL at 1732486472812Closed at 1732486472812 2024-11-24T22:14:32,813 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 967acf99df098c28d8f837a9f0746a90, NAME => 'test,lll,1732486472486.967acf99df098c28d8f837a9f0746a90.', STARTKEY => 'lll', ENDKEY => 'mmm'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c 2024-11-24T22:14:32,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34013 is added to blk_1073741846_1022 (size=45) 2024-11-24T22:14:32,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35383 is added to blk_1073741846_1022 (size=45) 2024-11-24T22:14:32,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741846_1022 (size=45) 2024-11-24T22:14:32,843 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,jjj,1732486472486.9b2e9ce57ffd0cd61c72602e5f6d9cae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:32,843 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 9b2e9ce57ffd0cd61c72602e5f6d9cae, disabling compactions & flushes 2024-11-24T22:14:32,843 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,jjj,1732486472486.9b2e9ce57ffd0cd61c72602e5f6d9cae. 2024-11-24T22:14:32,843 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1732486472486.9b2e9ce57ffd0cd61c72602e5f6d9cae. 2024-11-24T22:14:32,843 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1732486472486.9b2e9ce57ffd0cd61c72602e5f6d9cae. after waiting 0 ms 2024-11-24T22:14:32,843 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1732486472486.9b2e9ce57ffd0cd61c72602e5f6d9cae. 2024-11-24T22:14:32,844 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,jjj,1732486472486.9b2e9ce57ffd0cd61c72602e5f6d9cae. 2024-11-24T22:14:32,844 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9b2e9ce57ffd0cd61c72602e5f6d9cae: Waiting for close lock at 1732486472843Disabling compacts and flushes for region at 1732486472843Disabling writes for close at 1732486472843Writing region close event to WAL at 1732486472844 (+1 ms)Closed at 1732486472844 2024-11-24T22:14:32,844 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => dc147c653b22684b8e41092cd6a0c8eb, NAME => 'test,mmm,1732486472486.dc147c653b22684b8e41092cd6a0c8eb.', STARTKEY => 'mmm', ENDKEY => 'nnn'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c 2024-11-24T22:14:32,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35383 is added to blk_1073741848_1024 (size=45) 2024-11-24T22:14:32,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741848_1024 (size=45) 2024-11-24T22:14:32,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34013 is added to blk_1073741848_1024 (size=45) 2024-11-24T22:14:32,861 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,kkk,1732486472486.0b86957c7b81466a818bbff45fac751f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:32,861 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 0b86957c7b81466a818bbff45fac751f, disabling compactions & flushes 2024-11-24T22:14:32,861 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,kkk,1732486472486.0b86957c7b81466a818bbff45fac751f. 2024-11-24T22:14:32,861 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1732486472486.0b86957c7b81466a818bbff45fac751f. 2024-11-24T22:14:32,861 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1732486472486.0b86957c7b81466a818bbff45fac751f. after waiting 0 ms 2024-11-24T22:14:32,861 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1732486472486.0b86957c7b81466a818bbff45fac751f. 2024-11-24T22:14:32,861 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,kkk,1732486472486.0b86957c7b81466a818bbff45fac751f. 2024-11-24T22:14:32,862 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 0b86957c7b81466a818bbff45fac751f: Waiting for close lock at 1732486472861Disabling compacts and flushes for region at 1732486472861Disabling writes for close at 1732486472861Writing region close event to WAL at 1732486472861Closed at 1732486472861 2024-11-24T22:14:32,863 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 5160ff1dd0e56b08fbca2ffe581141d4, NAME => 'test,nnn,1732486472486.5160ff1dd0e56b08fbca2ffe581141d4.', STARTKEY => 'nnn', ENDKEY => 'ooo'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c 2024-11-24T22:14:32,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36279 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T22:14:32,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741849_1025 (size=45) 2024-11-24T22:14:32,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34013 is added to blk_1073741849_1025 (size=45) 2024-11-24T22:14:32,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35383 is added to blk_1073741849_1025 (size=45) 2024-11-24T22:14:32,872 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,lll,1732486472486.967acf99df098c28d8f837a9f0746a90.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:32,872 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 967acf99df098c28d8f837a9f0746a90, disabling compactions & flushes 2024-11-24T22:14:32,872 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,lll,1732486472486.967acf99df098c28d8f837a9f0746a90. 2024-11-24T22:14:32,872 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1732486472486.967acf99df098c28d8f837a9f0746a90. 2024-11-24T22:14:32,872 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,lll,1732486472486.967acf99df098c28d8f837a9f0746a90. after waiting 0 ms 2024-11-24T22:14:32,872 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,lll,1732486472486.967acf99df098c28d8f837a9f0746a90. 2024-11-24T22:14:32,872 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,lll,1732486472486.967acf99df098c28d8f837a9f0746a90. 2024-11-24T22:14:32,873 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 967acf99df098c28d8f837a9f0746a90: Waiting for close lock at 1732486472872Disabling compacts and flushes for region at 1732486472872Disabling writes for close at 1732486472872Writing region close event to WAL at 1732486472872Closed at 1732486472872 2024-11-24T22:14:32,873 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => ce1f0bcb3026cb1cbbd9e63c21b83f99, NAME => 'test,ooo,1732486472486.ce1f0bcb3026cb1cbbd9e63c21b83f99.', STARTKEY => 'ooo', ENDKEY => 'ppp'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c 2024-11-24T22:14:32,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35383 is added to blk_1073741850_1026 (size=45) 2024-11-24T22:14:32,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741850_1026 (size=45) 2024-11-24T22:14:32,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34013 is added to blk_1073741850_1026 (size=45) 2024-11-24T22:14:32,896 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,mmm,1732486472486.dc147c653b22684b8e41092cd6a0c8eb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:32,896 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing dc147c653b22684b8e41092cd6a0c8eb, disabling compactions & flushes 2024-11-24T22:14:32,896 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,mmm,1732486472486.dc147c653b22684b8e41092cd6a0c8eb. 2024-11-24T22:14:32,896 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1732486472486.dc147c653b22684b8e41092cd6a0c8eb. 2024-11-24T22:14:32,896 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1732486472486.dc147c653b22684b8e41092cd6a0c8eb. after waiting 0 ms 2024-11-24T22:14:32,896 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1732486472486.dc147c653b22684b8e41092cd6a0c8eb. 2024-11-24T22:14:32,896 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,mmm,1732486472486.dc147c653b22684b8e41092cd6a0c8eb. 2024-11-24T22:14:32,896 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for dc147c653b22684b8e41092cd6a0c8eb: Waiting for close lock at 1732486472896Disabling compacts and flushes for region at 1732486472896Disabling writes for close at 1732486472896Writing region close event to WAL at 1732486472896Closed at 1732486472896 2024-11-24T22:14:32,897 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => cca6a66154e5c299fad3a247afbfb5fd, NAME => 'test,ppp,1732486472486.cca6a66154e5c299fad3a247afbfb5fd.', STARTKEY => 'ppp', ENDKEY => 'qqq'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c 2024-11-24T22:14:32,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35383 is added to blk_1073741851_1027 (size=45) 2024-11-24T22:14:32,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741851_1027 (size=45) 2024-11-24T22:14:32,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34013 is added to blk_1073741851_1027 (size=45) 2024-11-24T22:14:32,903 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,nnn,1732486472486.5160ff1dd0e56b08fbca2ffe581141d4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:32,903 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 5160ff1dd0e56b08fbca2ffe581141d4, disabling compactions & flushes 2024-11-24T22:14:32,903 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,nnn,1732486472486.5160ff1dd0e56b08fbca2ffe581141d4. 2024-11-24T22:14:32,903 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1732486472486.5160ff1dd0e56b08fbca2ffe581141d4. 2024-11-24T22:14:32,903 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1732486472486.5160ff1dd0e56b08fbca2ffe581141d4. after waiting 0 ms 2024-11-24T22:14:32,903 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1732486472486.5160ff1dd0e56b08fbca2ffe581141d4. 2024-11-24T22:14:32,904 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,nnn,1732486472486.5160ff1dd0e56b08fbca2ffe581141d4. 2024-11-24T22:14:32,904 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 5160ff1dd0e56b08fbca2ffe581141d4: Waiting for close lock at 1732486472903Disabling compacts and flushes for region at 1732486472903Disabling writes for close at 1732486472903Writing region close event to WAL at 1732486472903Closed at 1732486472903 2024-11-24T22:14:32,904 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 83e58b0e0989e11c83283bf7bd41306c, NAME => 'test,qqq,1732486472486.83e58b0e0989e11c83283bf7bd41306c.', STARTKEY => 'qqq', ENDKEY => 'rrr'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c 2024-11-24T22:14:32,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34013 is added to blk_1073741852_1028 (size=45) 2024-11-24T22:14:32,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35383 is added to blk_1073741852_1028 (size=45) 2024-11-24T22:14:32,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741852_1028 (size=45) 2024-11-24T22:14:32,924 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ooo,1732486472486.ce1f0bcb3026cb1cbbd9e63c21b83f99.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:32,924 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing ce1f0bcb3026cb1cbbd9e63c21b83f99, disabling compactions & flushes 2024-11-24T22:14:32,924 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ooo,1732486472486.ce1f0bcb3026cb1cbbd9e63c21b83f99. 2024-11-24T22:14:32,924 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1732486472486.ce1f0bcb3026cb1cbbd9e63c21b83f99. 2024-11-24T22:14:32,924 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1732486472486.ce1f0bcb3026cb1cbbd9e63c21b83f99. after waiting 0 ms 2024-11-24T22:14:32,924 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1732486472486.ce1f0bcb3026cb1cbbd9e63c21b83f99. 2024-11-24T22:14:32,924 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ooo,1732486472486.ce1f0bcb3026cb1cbbd9e63c21b83f99. 2024-11-24T22:14:32,924 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for ce1f0bcb3026cb1cbbd9e63c21b83f99: Waiting for close lock at 1732486472924Disabling compacts and flushes for region at 1732486472924Disabling writes for close at 1732486472924Writing region close event to WAL at 1732486472924Closed at 1732486472924 2024-11-24T22:14:32,925 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => c802d82f160c5269119d007388afb85e, NAME => 'test,rrr,1732486472486.c802d82f160c5269119d007388afb85e.', STARTKEY => 'rrr', ENDKEY => 'sss'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c 2024-11-24T22:14:32,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741853_1029 (size=45) 2024-11-24T22:14:32,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35383 is added to blk_1073741853_1029 (size=45) 2024-11-24T22:14:32,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34013 is added to blk_1073741853_1029 (size=45) 2024-11-24T22:14:32,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34013 is added to blk_1073741854_1030 (size=45) 2024-11-24T22:14:32,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741854_1030 (size=45) 2024-11-24T22:14:32,935 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ppp,1732486472486.cca6a66154e5c299fad3a247afbfb5fd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:32,935 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing cca6a66154e5c299fad3a247afbfb5fd, disabling compactions & flushes 2024-11-24T22:14:32,935 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ppp,1732486472486.cca6a66154e5c299fad3a247afbfb5fd. 2024-11-24T22:14:32,935 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1732486472486.cca6a66154e5c299fad3a247afbfb5fd. 2024-11-24T22:14:32,935 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1732486472486.cca6a66154e5c299fad3a247afbfb5fd. after waiting 0 ms 2024-11-24T22:14:32,935 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1732486472486.cca6a66154e5c299fad3a247afbfb5fd. 2024-11-24T22:14:32,935 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ppp,1732486472486.cca6a66154e5c299fad3a247afbfb5fd. 2024-11-24T22:14:32,936 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for cca6a66154e5c299fad3a247afbfb5fd: Waiting for close lock at 1732486472935Disabling compacts and flushes for region at 1732486472935Disabling writes for close at 1732486472935Writing region close event to WAL at 1732486472935Closed at 1732486472935 2024-11-24T22:14:32,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35383 is added to blk_1073741854_1030 (size=45) 2024-11-24T22:14:32,936 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 84dd509902edc43083bc78fe4b70922c, NAME => 'test,sss,1732486472486.84dd509902edc43083bc78fe4b70922c.', STARTKEY => 'sss', ENDKEY => 'ttt'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c 2024-11-24T22:14:32,938 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,qqq,1732486472486.83e58b0e0989e11c83283bf7bd41306c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:32,938 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 83e58b0e0989e11c83283bf7bd41306c, disabling compactions & flushes 2024-11-24T22:14:32,938 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,qqq,1732486472486.83e58b0e0989e11c83283bf7bd41306c. 2024-11-24T22:14:32,939 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1732486472486.83e58b0e0989e11c83283bf7bd41306c. 2024-11-24T22:14:32,939 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1732486472486.83e58b0e0989e11c83283bf7bd41306c. after waiting 0 ms 2024-11-24T22:14:32,939 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1732486472486.83e58b0e0989e11c83283bf7bd41306c. 2024-11-24T22:14:32,939 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,qqq,1732486472486.83e58b0e0989e11c83283bf7bd41306c. 2024-11-24T22:14:32,939 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 83e58b0e0989e11c83283bf7bd41306c: Waiting for close lock at 1732486472938Disabling compacts and flushes for region at 1732486472938Disabling writes for close at 1732486472939 (+1 ms)Writing region close event to WAL at 1732486472939Closed at 1732486472939 2024-11-24T22:14:32,939 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => acdab7e426c93b10b34b4d331b21cef0, NAME => 'test,ttt,1732486472486.acdab7e426c93b10b34b4d331b21cef0.', STARTKEY => 'ttt', ENDKEY => 'uuu'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c 2024-11-24T22:14:32,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34013 is added to blk_1073741856_1032 (size=45) 2024-11-24T22:14:32,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35383 is added to blk_1073741856_1032 (size=45) 2024-11-24T22:14:32,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741856_1032 (size=45) 2024-11-24T22:14:32,982 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ttt,1732486472486.acdab7e426c93b10b34b4d331b21cef0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:32,982 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing acdab7e426c93b10b34b4d331b21cef0, disabling compactions & flushes 2024-11-24T22:14:32,982 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ttt,1732486472486.acdab7e426c93b10b34b4d331b21cef0. 2024-11-24T22:14:32,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35383 is added to blk_1073741857_1033 (size=45) 2024-11-24T22:14:32,982 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1732486472486.acdab7e426c93b10b34b4d331b21cef0. 2024-11-24T22:14:32,982 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1732486472486.acdab7e426c93b10b34b4d331b21cef0. after waiting 0 ms 2024-11-24T22:14:32,982 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1732486472486.acdab7e426c93b10b34b4d331b21cef0. 2024-11-24T22:14:32,982 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ttt,1732486472486.acdab7e426c93b10b34b4d331b21cef0. 2024-11-24T22:14:32,983 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for acdab7e426c93b10b34b4d331b21cef0: Waiting for close lock at 1732486472982Disabling compacts and flushes for region at 1732486472982Disabling writes for close at 1732486472982Writing region close event to WAL at 1732486472982Closed at 1732486472982 2024-11-24T22:14:32,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34013 is added to blk_1073741857_1033 (size=45) 2024-11-24T22:14:32,983 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 79c2dd6f654fabefe0fc06b8199e8562, NAME => 'test,uuu,1732486472486.79c2dd6f654fabefe0fc06b8199e8562.', STARTKEY => 'uuu', ENDKEY => 'vvv'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c 2024-11-24T22:14:32,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741857_1033 (size=45) 2024-11-24T22:14:32,993 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,sss,1732486472486.84dd509902edc43083bc78fe4b70922c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:32,994 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 84dd509902edc43083bc78fe4b70922c, disabling compactions & flushes 2024-11-24T22:14:32,994 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,sss,1732486472486.84dd509902edc43083bc78fe4b70922c. 2024-11-24T22:14:32,994 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1732486472486.84dd509902edc43083bc78fe4b70922c. 2024-11-24T22:14:32,994 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,sss,1732486472486.84dd509902edc43083bc78fe4b70922c. after waiting 0 ms 2024-11-24T22:14:32,994 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,sss,1732486472486.84dd509902edc43083bc78fe4b70922c. 2024-11-24T22:14:32,994 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,sss,1732486472486.84dd509902edc43083bc78fe4b70922c. 2024-11-24T22:14:32,994 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 84dd509902edc43083bc78fe4b70922c: Waiting for close lock at 1732486472994Disabling compacts and flushes for region at 1732486472994Disabling writes for close at 1732486472994Writing region close event to WAL at 1732486472994Closed at 1732486472994 2024-11-24T22:14:32,995 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => fc9f96fd4c44240a85e6b2c001e46153, NAME => 'test,vvv,1732486472486.fc9f96fd4c44240a85e6b2c001e46153.', STARTKEY => 'vvv', ENDKEY => 'www'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c 2024-11-24T22:14:33,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34013 is added to blk_1073741855_1031 (size=45) 2024-11-24T22:14:33,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35383 is added to blk_1073741855_1031 (size=45) 2024-11-24T22:14:33,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741855_1031 (size=45) 2024-11-24T22:14:33,041 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,rrr,1732486472486.c802d82f160c5269119d007388afb85e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:33,042 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing c802d82f160c5269119d007388afb85e, disabling compactions & flushes 2024-11-24T22:14:33,042 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,rrr,1732486472486.c802d82f160c5269119d007388afb85e. 2024-11-24T22:14:33,042 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1732486472486.c802d82f160c5269119d007388afb85e. 2024-11-24T22:14:33,042 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1732486472486.c802d82f160c5269119d007388afb85e. after waiting 0 ms 2024-11-24T22:14:33,042 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1732486472486.c802d82f160c5269119d007388afb85e. 2024-11-24T22:14:33,042 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,rrr,1732486472486.c802d82f160c5269119d007388afb85e. 2024-11-24T22:14:33,042 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for c802d82f160c5269119d007388afb85e: Waiting for close lock at 1732486473041Disabling compacts and flushes for region at 1732486473041Disabling writes for close at 1732486473042 (+1 ms)Writing region close event to WAL at 1732486473042Closed at 1732486473042 2024-11-24T22:14:33,043 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 50d9b38af2d6413e500860133d150aa8, NAME => 'test,www,1732486472486.50d9b38af2d6413e500860133d150aa8.', STARTKEY => 'www', ENDKEY => 'xxx'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c 2024-11-24T22:14:33,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35383 is added to blk_1073741858_1034 (size=45) 2024-11-24T22:14:33,067 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,uuu,1732486472486.79c2dd6f654fabefe0fc06b8199e8562.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:33,067 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 79c2dd6f654fabefe0fc06b8199e8562, disabling compactions & flushes 2024-11-24T22:14:33,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741858_1034 (size=45) 2024-11-24T22:14:33,067 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,uuu,1732486472486.79c2dd6f654fabefe0fc06b8199e8562. 2024-11-24T22:14:33,067 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1732486472486.79c2dd6f654fabefe0fc06b8199e8562. 2024-11-24T22:14:33,067 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1732486472486.79c2dd6f654fabefe0fc06b8199e8562. after waiting 0 ms 2024-11-24T22:14:33,067 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1732486472486.79c2dd6f654fabefe0fc06b8199e8562. 2024-11-24T22:14:33,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34013 is added to blk_1073741858_1034 (size=45) 2024-11-24T22:14:33,068 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,uuu,1732486472486.79c2dd6f654fabefe0fc06b8199e8562. 2024-11-24T22:14:33,068 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 79c2dd6f654fabefe0fc06b8199e8562: Waiting for close lock at 1732486473067Disabling compacts and flushes for region at 1732486473067Disabling writes for close at 1732486473067Writing region close event to WAL at 1732486473067Closed at 1732486473068 (+1 ms) 2024-11-24T22:14:33,069 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 516ead6fe1773c4da8708cdc5713f526, NAME => 'test,xxx,1732486472486.516ead6fe1773c4da8708cdc5713f526.', STARTKEY => 'xxx', ENDKEY => 'yyy'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c 2024-11-24T22:14:33,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34013 is added to blk_1073741859_1035 (size=45) 2024-11-24T22:14:33,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35383 is added to blk_1073741859_1035 (size=45) 2024-11-24T22:14:33,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741859_1035 (size=45) 2024-11-24T22:14:33,076 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,vvv,1732486472486.fc9f96fd4c44240a85e6b2c001e46153.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:33,076 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing fc9f96fd4c44240a85e6b2c001e46153, disabling compactions & flushes 2024-11-24T22:14:33,077 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,vvv,1732486472486.fc9f96fd4c44240a85e6b2c001e46153. 2024-11-24T22:14:33,077 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1732486472486.fc9f96fd4c44240a85e6b2c001e46153. 2024-11-24T22:14:33,077 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1732486472486.fc9f96fd4c44240a85e6b2c001e46153. after waiting 0 ms 2024-11-24T22:14:33,077 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1732486472486.fc9f96fd4c44240a85e6b2c001e46153. 2024-11-24T22:14:33,077 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,vvv,1732486472486.fc9f96fd4c44240a85e6b2c001e46153. 2024-11-24T22:14:33,077 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for fc9f96fd4c44240a85e6b2c001e46153: Waiting for close lock at 1732486473076Disabling compacts and flushes for region at 1732486473076Disabling writes for close at 1732486473077 (+1 ms)Writing region close event to WAL at 1732486473077Closed at 1732486473077 2024-11-24T22:14:33,078 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 79d315a497ec3ad891988cb800bae337, NAME => 'test,yyy,1732486472486.79d315a497ec3ad891988cb800bae337.', STARTKEY => 'yyy', ENDKEY => 'zzz'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c 2024-11-24T22:14:33,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34013 is added to blk_1073741860_1036 (size=45) 2024-11-24T22:14:33,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741860_1036 (size=45) 2024-11-24T22:14:33,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35383 is added to blk_1073741860_1036 (size=45) 2024-11-24T22:14:33,088 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,www,1732486472486.50d9b38af2d6413e500860133d150aa8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:33,089 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 50d9b38af2d6413e500860133d150aa8, disabling compactions & flushes 2024-11-24T22:14:33,089 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,www,1732486472486.50d9b38af2d6413e500860133d150aa8. 2024-11-24T22:14:33,089 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1732486472486.50d9b38af2d6413e500860133d150aa8. 2024-11-24T22:14:33,089 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,www,1732486472486.50d9b38af2d6413e500860133d150aa8. after waiting 0 ms 2024-11-24T22:14:33,089 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,www,1732486472486.50d9b38af2d6413e500860133d150aa8. 2024-11-24T22:14:33,089 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,www,1732486472486.50d9b38af2d6413e500860133d150aa8. 2024-11-24T22:14:33,089 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 50d9b38af2d6413e500860133d150aa8: Waiting for close lock at 1732486473089Disabling compacts and flushes for region at 1732486473089Disabling writes for close at 1732486473089Writing region close event to WAL at 1732486473089Closed at 1732486473089 2024-11-24T22:14:33,090 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => ed5b70c013e388180062b1513b980a07, NAME => 'test,zzz,1732486472486.ed5b70c013e388180062b1513b980a07.', STARTKEY => 'zzz', ENDKEY => ''}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c 2024-11-24T22:14:33,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34013 is added to blk_1073741861_1037 (size=45) 2024-11-24T22:14:33,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741861_1037 (size=45) 2024-11-24T22:14:33,113 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,yyy,1732486472486.79d315a497ec3ad891988cb800bae337.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:33,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35383 is added to blk_1073741861_1037 (size=45) 2024-11-24T22:14:33,114 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 79d315a497ec3ad891988cb800bae337, disabling compactions & flushes 2024-11-24T22:14:33,114 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,yyy,1732486472486.79d315a497ec3ad891988cb800bae337. 2024-11-24T22:14:33,114 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1732486472486.79d315a497ec3ad891988cb800bae337. 2024-11-24T22:14:33,114 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1732486472486.79d315a497ec3ad891988cb800bae337. after waiting 0 ms 2024-11-24T22:14:33,114 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1732486472486.79d315a497ec3ad891988cb800bae337. 2024-11-24T22:14:33,114 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,yyy,1732486472486.79d315a497ec3ad891988cb800bae337. 2024-11-24T22:14:33,114 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 79d315a497ec3ad891988cb800bae337: Waiting for close lock at 1732486473113Disabling compacts and flushes for region at 1732486473113Disabling writes for close at 1732486473114 (+1 ms)Writing region close event to WAL at 1732486473114Closed at 1732486473114 2024-11-24T22:14:33,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34013 is added to blk_1073741862_1038 (size=45) 2024-11-24T22:14:33,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35383 is added to blk_1073741862_1038 (size=45) 2024-11-24T22:14:33,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741862_1038 (size=45) 2024-11-24T22:14:33,139 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,xxx,1732486472486.516ead6fe1773c4da8708cdc5713f526.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:33,139 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 516ead6fe1773c4da8708cdc5713f526, disabling compactions & flushes 2024-11-24T22:14:33,140 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,xxx,1732486472486.516ead6fe1773c4da8708cdc5713f526. 2024-11-24T22:14:33,140 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1732486472486.516ead6fe1773c4da8708cdc5713f526. 2024-11-24T22:14:33,140 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1732486472486.516ead6fe1773c4da8708cdc5713f526. after waiting 0 ms 2024-11-24T22:14:33,140 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1732486472486.516ead6fe1773c4da8708cdc5713f526. 2024-11-24T22:14:33,140 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,xxx,1732486472486.516ead6fe1773c4da8708cdc5713f526. 2024-11-24T22:14:33,140 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 516ead6fe1773c4da8708cdc5713f526: Waiting for close lock at 1732486473139Disabling compacts and flushes for region at 1732486473139Disabling writes for close at 1732486473140 (+1 ms)Writing region close event to WAL at 1732486473140Closed at 1732486473140 2024-11-24T22:14:33,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34013 is added to blk_1073741863_1039 (size=42) 2024-11-24T22:14:33,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35383 is added to blk_1073741863_1039 (size=42) 2024-11-24T22:14:33,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741863_1039 (size=42) 2024-11-24T22:14:33,157 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,zzz,1732486472486.ed5b70c013e388180062b1513b980a07.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:33,157 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing ed5b70c013e388180062b1513b980a07, disabling compactions & flushes 2024-11-24T22:14:33,157 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,zzz,1732486472486.ed5b70c013e388180062b1513b980a07. 2024-11-24T22:14:33,157 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1732486472486.ed5b70c013e388180062b1513b980a07. 2024-11-24T22:14:33,157 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1732486472486.ed5b70c013e388180062b1513b980a07. after waiting 0 ms 2024-11-24T22:14:33,157 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1732486472486.ed5b70c013e388180062b1513b980a07. 2024-11-24T22:14:33,157 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,zzz,1732486472486.ed5b70c013e388180062b1513b980a07. 2024-11-24T22:14:33,158 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for ed5b70c013e388180062b1513b980a07: Waiting for close lock at 1732486473157Disabling compacts and flushes for region at 1732486473157Disabling writes for close at 1732486473157Writing region close event to WAL at 1732486473157Closed at 1732486473157 2024-11-24T22:14:33,161 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T22:14:33,168 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,,1732486472486.4e91326d9778777879bc895ecb241af8.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1732486473162"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732486473162"}]},"ts":"1732486473162"} 2024-11-24T22:14:33,168 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ccc,1732486472486.c68a44a11eeaccf8a3c2f49ba1446c1e.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732486473162"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732486473162"}]},"ts":"1732486473162"} 2024-11-24T22:14:33,168 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,bbb,1732486472486.4279880609a62ccb9e9fb29e79bdf7e7.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732486473162"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732486473162"}]},"ts":"1732486473162"} 2024-11-24T22:14:33,168 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ddd,1732486472486.698dd9d64bdaf6a81e797d2590ff86f9.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732486473162"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732486473162"}]},"ts":"1732486473162"} 2024-11-24T22:14:33,169 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,eee,1732486472486.46b79f5a3db63445091682b5e72f635a.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732486473162"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732486473162"}]},"ts":"1732486473162"} 2024-11-24T22:14:33,169 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,fff,1732486472486.736f44b8f348e2a5a96dfd3abc5e62b2.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732486473162"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732486473162"}]},"ts":"1732486473162"} 2024-11-24T22:14:33,169 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ggg,1732486472486.f72c935213180e3ff989af795022b7e4.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732486473162"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732486473162"}]},"ts":"1732486473162"} 2024-11-24T22:14:33,169 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,hhh,1732486472486.72a41015f7b88194cc3a08feb66aa4f9.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732486473162"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732486473162"}]},"ts":"1732486473162"} 2024-11-24T22:14:33,169 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,iii,1732486472486.94f2fa48145fe3bdeffbfded6a4edd73.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732486473162"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732486473162"}]},"ts":"1732486473162"} 2024-11-24T22:14:33,170 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,jjj,1732486472486.9b2e9ce57ffd0cd61c72602e5f6d9cae.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732486473162"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732486473162"}]},"ts":"1732486473162"} 2024-11-24T22:14:33,170 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,kkk,1732486472486.0b86957c7b81466a818bbff45fac751f.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732486473162"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732486473162"}]},"ts":"1732486473162"} 2024-11-24T22:14:33,170 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,lll,1732486472486.967acf99df098c28d8f837a9f0746a90.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732486473162"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732486473162"}]},"ts":"1732486473162"} 2024-11-24T22:14:33,170 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,mmm,1732486472486.dc147c653b22684b8e41092cd6a0c8eb.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732486473162"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732486473162"}]},"ts":"1732486473162"} 2024-11-24T22:14:33,170 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,nnn,1732486472486.5160ff1dd0e56b08fbca2ffe581141d4.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732486473162"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732486473162"}]},"ts":"1732486473162"} 2024-11-24T22:14:33,171 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ooo,1732486472486.ce1f0bcb3026cb1cbbd9e63c21b83f99.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732486473162"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732486473162"}]},"ts":"1732486473162"} 2024-11-24T22:14:33,171 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ppp,1732486472486.cca6a66154e5c299fad3a247afbfb5fd.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732486473162"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732486473162"}]},"ts":"1732486473162"} 2024-11-24T22:14:33,171 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,qqq,1732486472486.83e58b0e0989e11c83283bf7bd41306c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732486473162"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732486473162"}]},"ts":"1732486473162"} 2024-11-24T22:14:33,171 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ttt,1732486472486.acdab7e426c93b10b34b4d331b21cef0.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732486473162"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732486473162"}]},"ts":"1732486473162"} 2024-11-24T22:14:33,171 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,sss,1732486472486.84dd509902edc43083bc78fe4b70922c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732486473162"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732486473162"}]},"ts":"1732486473162"} 2024-11-24T22:14:33,172 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,rrr,1732486472486.c802d82f160c5269119d007388afb85e.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732486473162"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732486473162"}]},"ts":"1732486473162"} 2024-11-24T22:14:33,172 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,uuu,1732486472486.79c2dd6f654fabefe0fc06b8199e8562.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732486473162"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732486473162"}]},"ts":"1732486473162"} 2024-11-24T22:14:33,172 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,vvv,1732486472486.fc9f96fd4c44240a85e6b2c001e46153.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732486473162"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732486473162"}]},"ts":"1732486473162"} 2024-11-24T22:14:33,172 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,www,1732486472486.50d9b38af2d6413e500860133d150aa8.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732486473162"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732486473162"}]},"ts":"1732486473162"} 2024-11-24T22:14:33,172 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,yyy,1732486472486.79d315a497ec3ad891988cb800bae337.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732486473162"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732486473162"}]},"ts":"1732486473162"} 2024-11-24T22:14:33,173 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,xxx,1732486472486.516ead6fe1773c4da8708cdc5713f526.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732486473162"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732486473162"}]},"ts":"1732486473162"} 2024-11-24T22:14:33,173 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,zzz,1732486472486.ed5b70c013e388180062b1513b980a07.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1732486473162"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732486473162"}]},"ts":"1732486473162"} 2024-11-24T22:14:33,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36279 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T22:14:33,243 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 26 regions to meta. 2024-11-24T22:14:33,246 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T22:14:33,249 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732486473246"}]},"ts":"1732486473246"} 2024-11-24T22:14:33,256 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLING in hbase:meta 2024-11-24T22:14:33,258 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {bd6592c37bb2=0} racks are {/default-rack=0} 2024-11-24T22:14:33,285 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T22:14:33,285 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T22:14:33,285 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-24T22:14:33,285 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T22:14:33,285 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T22:14:33,285 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-24T22:14:33,285 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T22:14:33,285 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T22:14:33,286 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-24T22:14:33,286 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T22:14:33,290 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4e91326d9778777879bc895ecb241af8, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4279880609a62ccb9e9fb29e79bdf7e7, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c68a44a11eeaccf8a3c2f49ba1446c1e, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=46b79f5a3db63445091682b5e72f635a, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=736f44b8f348e2a5a96dfd3abc5e62b2, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f72c935213180e3ff989af795022b7e4, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=72a41015f7b88194cc3a08feb66aa4f9, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=94f2fa48145fe3bdeffbfded6a4edd73, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae, ASSIGN}, {pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0b86957c7b81466a818bbff45fac751f, ASSIGN}, {pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=967acf99df098c28d8f837a9f0746a90, ASSIGN}, {pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN}, {pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5160ff1dd0e56b08fbca2ffe581141d4, ASSIGN}, {pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ce1f0bcb3026cb1cbbd9e63c21b83f99, ASSIGN}, {pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=cca6a66154e5c299fad3a247afbfb5fd, ASSIGN}, {pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=83e58b0e0989e11c83283bf7bd41306c, ASSIGN}, {pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c802d82f160c5269119d007388afb85e, ASSIGN}, {pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=84dd509902edc43083bc78fe4b70922c, ASSIGN}, {pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=acdab7e426c93b10b34b4d331b21cef0, ASSIGN}, {pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=79c2dd6f654fabefe0fc06b8199e8562, ASSIGN}, {pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN}, {pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=50d9b38af2d6413e500860133d150aa8, ASSIGN}, {pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=516ead6fe1773c4da8708cdc5713f526, ASSIGN}, {pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=79d315a497ec3ad891988cb800bae337, ASSIGN}, {pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ed5b70c013e388180062b1513b980a07, ASSIGN}] 2024-11-24T22:14:33,303 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ed5b70c013e388180062b1513b980a07, ASSIGN 2024-11-24T22:14:33,305 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ed5b70c013e388180062b1513b980a07, ASSIGN; state=OFFLINE, location=bd6592c37bb2,42639,1732486469476; forceNewPlan=false, retain=false 2024-11-24T22:14:33,307 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=79d315a497ec3ad891988cb800bae337, ASSIGN 2024-11-24T22:14:33,311 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=79d315a497ec3ad891988cb800bae337, ASSIGN; state=OFFLINE, location=bd6592c37bb2,38125,1732486469405; forceNewPlan=false, retain=false 2024-11-24T22:14:33,313 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=516ead6fe1773c4da8708cdc5713f526, ASSIGN 2024-11-24T22:14:33,315 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=516ead6fe1773c4da8708cdc5713f526, ASSIGN; state=OFFLINE, location=bd6592c37bb2,41619,1732486469269; forceNewPlan=false, retain=false 2024-11-24T22:14:33,317 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=50d9b38af2d6413e500860133d150aa8, ASSIGN 2024-11-24T22:14:33,319 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=50d9b38af2d6413e500860133d150aa8, ASSIGN; state=OFFLINE, location=bd6592c37bb2,42639,1732486469476; forceNewPlan=false, retain=false 2024-11-24T22:14:33,321 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN 2024-11-24T22:14:33,323 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN; state=OFFLINE, location=bd6592c37bb2,41619,1732486469269; forceNewPlan=false, retain=false 2024-11-24T22:14:33,328 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=79c2dd6f654fabefe0fc06b8199e8562, ASSIGN 2024-11-24T22:14:33,330 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=79c2dd6f654fabefe0fc06b8199e8562, ASSIGN; state=OFFLINE, location=bd6592c37bb2,38125,1732486469405; forceNewPlan=false, retain=false 2024-11-24T22:14:33,333 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=acdab7e426c93b10b34b4d331b21cef0, ASSIGN 2024-11-24T22:14:33,335 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=acdab7e426c93b10b34b4d331b21cef0, ASSIGN; state=OFFLINE, location=bd6592c37bb2,38125,1732486469405; forceNewPlan=false, retain=false 2024-11-24T22:14:33,338 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=84dd509902edc43083bc78fe4b70922c, ASSIGN 2024-11-24T22:14:33,342 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=84dd509902edc43083bc78fe4b70922c, ASSIGN; state=OFFLINE, location=bd6592c37bb2,41619,1732486469269; forceNewPlan=false, retain=false 2024-11-24T22:14:33,344 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c802d82f160c5269119d007388afb85e, ASSIGN 2024-11-24T22:14:33,345 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c802d82f160c5269119d007388afb85e, ASSIGN; state=OFFLINE, location=bd6592c37bb2,42639,1732486469476; forceNewPlan=false, retain=false 2024-11-24T22:14:33,347 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=83e58b0e0989e11c83283bf7bd41306c, ASSIGN 2024-11-24T22:14:33,351 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=83e58b0e0989e11c83283bf7bd41306c, ASSIGN; state=OFFLINE, location=bd6592c37bb2,42639,1732486469476; forceNewPlan=false, retain=false 2024-11-24T22:14:33,354 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=cca6a66154e5c299fad3a247afbfb5fd, ASSIGN 2024-11-24T22:14:33,356 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=cca6a66154e5c299fad3a247afbfb5fd, ASSIGN; state=OFFLINE, location=bd6592c37bb2,41619,1732486469269; forceNewPlan=false, retain=false 2024-11-24T22:14:33,358 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ce1f0bcb3026cb1cbbd9e63c21b83f99, ASSIGN 2024-11-24T22:14:33,360 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ce1f0bcb3026cb1cbbd9e63c21b83f99, ASSIGN; state=OFFLINE, location=bd6592c37bb2,38125,1732486469405; forceNewPlan=false, retain=false 2024-11-24T22:14:33,362 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5160ff1dd0e56b08fbca2ffe581141d4, ASSIGN 2024-11-24T22:14:33,368 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=5160ff1dd0e56b08fbca2ffe581141d4, ASSIGN; state=OFFLINE, location=bd6592c37bb2,42639,1732486469476; forceNewPlan=false, retain=false 2024-11-24T22:14:33,370 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN 2024-11-24T22:14:33,374 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN; state=OFFLINE, location=bd6592c37bb2,41619,1732486469269; forceNewPlan=false, retain=false 2024-11-24T22:14:33,378 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=967acf99df098c28d8f837a9f0746a90, ASSIGN 2024-11-24T22:14:33,380 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=967acf99df098c28d8f837a9f0746a90, ASSIGN; state=OFFLINE, location=bd6592c37bb2,38125,1732486469405; forceNewPlan=false, retain=false 2024-11-24T22:14:33,382 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0b86957c7b81466a818bbff45fac751f, ASSIGN 2024-11-24T22:14:33,383 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=0b86957c7b81466a818bbff45fac751f, ASSIGN; state=OFFLINE, location=bd6592c37bb2,42639,1732486469476; forceNewPlan=false, retain=false 2024-11-24T22:14:33,387 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae, ASSIGN 2024-11-24T22:14:33,392 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae, ASSIGN; state=OFFLINE, location=bd6592c37bb2,41619,1732486469269; forceNewPlan=false, retain=false 2024-11-24T22:14:33,394 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=94f2fa48145fe3bdeffbfded6a4edd73, ASSIGN 2024-11-24T22:14:33,396 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=94f2fa48145fe3bdeffbfded6a4edd73, ASSIGN; state=OFFLINE, location=bd6592c37bb2,38125,1732486469405; forceNewPlan=false, retain=false 2024-11-24T22:14:33,398 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=72a41015f7b88194cc3a08feb66aa4f9, ASSIGN 2024-11-24T22:14:33,400 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=72a41015f7b88194cc3a08feb66aa4f9, ASSIGN; state=OFFLINE, location=bd6592c37bb2,42639,1732486469476; forceNewPlan=false, retain=false 2024-11-24T22:14:33,402 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f72c935213180e3ff989af795022b7e4, ASSIGN 2024-11-24T22:14:33,413 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f72c935213180e3ff989af795022b7e4, ASSIGN; state=OFFLINE, location=bd6592c37bb2,41619,1732486469269; forceNewPlan=false, retain=false 2024-11-24T22:14:33,424 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=736f44b8f348e2a5a96dfd3abc5e62b2, ASSIGN 2024-11-24T22:14:33,428 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=736f44b8f348e2a5a96dfd3abc5e62b2, ASSIGN; state=OFFLINE, location=bd6592c37bb2,38125,1732486469405; forceNewPlan=false, retain=false 2024-11-24T22:14:33,431 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=46b79f5a3db63445091682b5e72f635a, ASSIGN 2024-11-24T22:14:33,433 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=46b79f5a3db63445091682b5e72f635a, ASSIGN; state=OFFLINE, location=bd6592c37bb2,42639,1732486469476; forceNewPlan=false, retain=false 2024-11-24T22:14:33,437 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN 2024-11-24T22:14:33,442 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN; state=OFFLINE, location=bd6592c37bb2,41619,1732486469269; forceNewPlan=false, retain=false 2024-11-24T22:14:33,448 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c68a44a11eeaccf8a3c2f49ba1446c1e, ASSIGN 2024-11-24T22:14:33,453 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c68a44a11eeaccf8a3c2f49ba1446c1e, ASSIGN; state=OFFLINE, location=bd6592c37bb2,42639,1732486469476; forceNewPlan=false, retain=false 2024-11-24T22:14:33,455 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4279880609a62ccb9e9fb29e79bdf7e7, ASSIGN 2024-11-24T22:14:33,457 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4279880609a62ccb9e9fb29e79bdf7e7, ASSIGN; state=OFFLINE, location=bd6592c37bb2,38125,1732486469405; forceNewPlan=false, retain=false 2024-11-24T22:14:33,460 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4e91326d9778777879bc895ecb241af8, ASSIGN 2024-11-24T22:14:33,461 INFO [bd6592c37bb2:36279 {}] balancer.BaseLoadBalancer(388): Reassigned 24 regions. 24 retained the pre-restart assignment. 2024-11-24T22:14:33,463 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4e91326d9778777879bc895ecb241af8, ASSIGN; state=OFFLINE, location=bd6592c37bb2,41619,1732486469269; forceNewPlan=false, retain=false 2024-11-24T22:14:33,470 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=ed5b70c013e388180062b1513b980a07, regionState=OPENING, regionLocation=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:33,472 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=50d9b38af2d6413e500860133d150aa8, regionState=OPENING, regionLocation=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:33,473 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=c802d82f160c5269119d007388afb85e, regionState=OPENING, regionLocation=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:33,476 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=83e58b0e0989e11c83283bf7bd41306c, regionState=OPENING, regionLocation=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:33,478 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=5160ff1dd0e56b08fbca2ffe581141d4, regionState=OPENING, regionLocation=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:33,479 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ed5b70c013e388180062b1513b980a07, ASSIGN because future has completed 2024-11-24T22:14:33,481 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=0b86957c7b81466a818bbff45fac751f, regionState=OPENING, regionLocation=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:33,482 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=72a41015f7b88194cc3a08feb66aa4f9, regionState=OPENING, regionLocation=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:33,483 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=46b79f5a3db63445091682b5e72f635a, regionState=OPENING, regionLocation=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:33,483 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=c68a44a11eeaccf8a3c2f49ba1446c1e, regionState=OPENING, regionLocation=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:33,484 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=516ead6fe1773c4da8708cdc5713f526, regionState=OPENING, regionLocation=bd6592c37bb2,41619,1732486469269 2024-11-24T22:14:33,485 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=fc9f96fd4c44240a85e6b2c001e46153, regionState=OPENING, regionLocation=bd6592c37bb2,41619,1732486469269 2024-11-24T22:14:33,486 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=84dd509902edc43083bc78fe4b70922c, regionState=OPENING, regionLocation=bd6592c37bb2,41619,1732486469269 2024-11-24T22:14:33,486 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=cca6a66154e5c299fad3a247afbfb5fd, regionState=OPENING, regionLocation=bd6592c37bb2,41619,1732486469269 2024-11-24T22:14:33,487 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=dc147c653b22684b8e41092cd6a0c8eb, regionState=OPENING, regionLocation=bd6592c37bb2,41619,1732486469269 2024-11-24T22:14:33,488 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=9b2e9ce57ffd0cd61c72602e5f6d9cae, regionState=OPENING, regionLocation=bd6592c37bb2,41619,1732486469269 2024-11-24T22:14:33,488 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=f72c935213180e3ff989af795022b7e4, regionState=OPENING, regionLocation=bd6592c37bb2,41619,1732486469269 2024-11-24T22:14:33,489 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=698dd9d64bdaf6a81e797d2590ff86f9, regionState=OPENING, regionLocation=bd6592c37bb2,41619,1732486469269 2024-11-24T22:14:33,490 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=79d315a497ec3ad891988cb800bae337, regionState=OPENING, regionLocation=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:33,505 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=50d9b38af2d6413e500860133d150aa8, ASSIGN because future has completed 2024-11-24T22:14:33,505 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure ed5b70c013e388180062b1513b980a07, server=bd6592c37bb2,42639,1732486469476}] 2024-11-24T22:14:33,512 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c802d82f160c5269119d007388afb85e, ASSIGN because future has completed 2024-11-24T22:14:33,514 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=83e58b0e0989e11c83283bf7bd41306c, ASSIGN because future has completed 2024-11-24T22:14:33,522 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=5160ff1dd0e56b08fbca2ffe581141d4, ASSIGN because future has completed 2024-11-24T22:14:33,534 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=0b86957c7b81466a818bbff45fac751f, ASSIGN because future has completed 2024-11-24T22:14:33,536 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5160ff1dd0e56b08fbca2ffe581141d4, server=bd6592c37bb2,42639,1732486469476}] 2024-11-24T22:14:33,538 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=72a41015f7b88194cc3a08feb66aa4f9, ASSIGN because future has completed 2024-11-24T22:14:33,546 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=46b79f5a3db63445091682b5e72f635a, ASSIGN because future has completed 2024-11-24T22:14:33,553 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c68a44a11eeaccf8a3c2f49ba1446c1e, ASSIGN because future has completed 2024-11-24T22:14:33,554 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure c68a44a11eeaccf8a3c2f49ba1446c1e, server=bd6592c37bb2,42639,1732486469476}] 2024-11-24T22:14:33,555 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=516ead6fe1773c4da8708cdc5713f526, ASSIGN because future has completed 2024-11-24T22:14:33,557 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN because future has completed 2024-11-24T22:14:33,561 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=84dd509902edc43083bc78fe4b70922c, ASSIGN because future has completed 2024-11-24T22:14:33,562 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=cca6a66154e5c299fad3a247afbfb5fd, ASSIGN because future has completed 2024-11-24T22:14:33,563 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure cca6a66154e5c299fad3a247afbfb5fd, server=bd6592c37bb2,41619,1732486469269}] 2024-11-24T22:14:33,566 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN because future has completed 2024-11-24T22:14:33,570 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae, ASSIGN because future has completed 2024-11-24T22:14:33,572 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f72c935213180e3ff989af795022b7e4, ASSIGN because future has completed 2024-11-24T22:14:33,575 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure f72c935213180e3ff989af795022b7e4, server=bd6592c37bb2,41619,1732486469269}] 2024-11-24T22:14:33,576 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN because future has completed 2024-11-24T22:14:33,580 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=79d315a497ec3ad891988cb800bae337, ASSIGN because future has completed 2024-11-24T22:14:33,585 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 79d315a497ec3ad891988cb800bae337, server=bd6592c37bb2,38125,1732486469405}] 2024-11-24T22:14:33,597 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 698dd9d64bdaf6a81e797d2590ff86f9, server=bd6592c37bb2,41619,1732486469269}] 2024-11-24T22:14:33,605 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9b2e9ce57ffd0cd61c72602e5f6d9cae, server=bd6592c37bb2,41619,1732486469269}] 2024-11-24T22:14:33,613 INFO [bd6592c37bb2:36279 {}] balancer.BaseLoadBalancer(388): Reassigned 2 regions. 2 retained the pre-restart assignment. 2024-11-24T22:14:33,617 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=4e91326d9778777879bc895ecb241af8, regionState=OPENING, regionLocation=bd6592c37bb2,41619,1732486469269 2024-11-24T22:14:33,618 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=4279880609a62ccb9e9fb29e79bdf7e7, regionState=OPENING, regionLocation=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:33,618 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure dc147c653b22684b8e41092cd6a0c8eb, server=bd6592c37bb2,41619,1732486469269}] 2024-11-24T22:14:33,622 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4e91326d9778777879bc895ecb241af8, ASSIGN because future has completed 2024-11-24T22:14:33,624 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4279880609a62ccb9e9fb29e79bdf7e7, ASSIGN because future has completed 2024-11-24T22:14:33,629 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4279880609a62ccb9e9fb29e79bdf7e7, server=bd6592c37bb2,38125,1732486469405}] 2024-11-24T22:14:33,634 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e91326d9778777879bc895ecb241af8, server=bd6592c37bb2,41619,1732486469269}] 2024-11-24T22:14:33,643 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,41619,1732486469269}] 2024-11-24T22:14:33,650 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure fc9f96fd4c44240a85e6b2c001e46153, server=bd6592c37bb2,41619,1732486469269}] 2024-11-24T22:14:33,660 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 516ead6fe1773c4da8708cdc5713f526, server=bd6592c37bb2,41619,1732486469269}] 2024-11-24T22:14:33,665 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 46b79f5a3db63445091682b5e72f635a, server=bd6592c37bb2,42639,1732486469476}] 2024-11-24T22:14:33,666 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T22:14:33,669 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53075, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T22:14:33,675 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 72a41015f7b88194cc3a08feb66aa4f9, server=bd6592c37bb2,42639,1732486469476}] 2024-11-24T22:14:33,681 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open test,nnn,1732486472486.5160ff1dd0e56b08fbca2ffe581141d4. 2024-11-24T22:14:33,681 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 5160ff1dd0e56b08fbca2ffe581141d4, NAME => 'test,nnn,1732486472486.5160ff1dd0e56b08fbca2ffe581141d4.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-24T22:14:33,682 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0b86957c7b81466a818bbff45fac751f, server=bd6592c37bb2,42639,1732486469476}] 2024-11-24T22:14:33,682 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 5160ff1dd0e56b08fbca2ffe581141d4 2024-11-24T22:14:33,682 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated test,nnn,1732486472486.5160ff1dd0e56b08fbca2ffe581141d4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:33,683 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 5160ff1dd0e56b08fbca2ffe581141d4 2024-11-24T22:14:33,683 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 5160ff1dd0e56b08fbca2ffe581141d4 2024-11-24T22:14:33,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36279 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T22:14:33,687 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 83e58b0e0989e11c83283bf7bd41306c, server=bd6592c37bb2,42639,1732486469476}] 2024-11-24T22:14:33,688 INFO [StoreOpener-5160ff1dd0e56b08fbca2ffe581141d4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 5160ff1dd0e56b08fbca2ffe581141d4 2024-11-24T22:14:33,691 INFO [StoreOpener-5160ff1dd0e56b08fbca2ffe581141d4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5160ff1dd0e56b08fbca2ffe581141d4 columnFamilyName family 2024-11-24T22:14:33,691 DEBUG [StoreOpener-5160ff1dd0e56b08fbca2ffe581141d4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:33,692 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure c802d82f160c5269119d007388afb85e, server=bd6592c37bb2,42639,1732486469476}] 2024-11-24T22:14:33,693 INFO [StoreOpener-5160ff1dd0e56b08fbca2ffe581141d4-1 {}] regionserver.HStore(327): Store=5160ff1dd0e56b08fbca2ffe581141d4/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:14:33,693 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 5160ff1dd0e56b08fbca2ffe581141d4 2024-11-24T22:14:33,695 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/5160ff1dd0e56b08fbca2ffe581141d4 2024-11-24T22:14:33,695 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/5160ff1dd0e56b08fbca2ffe581141d4 2024-11-24T22:14:33,696 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 5160ff1dd0e56b08fbca2ffe581141d4 2024-11-24T22:14:33,696 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 5160ff1dd0e56b08fbca2ffe581141d4 2024-11-24T22:14:33,701 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 5160ff1dd0e56b08fbca2ffe581141d4 2024-11-24T22:14:33,702 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 50d9b38af2d6413e500860133d150aa8, server=bd6592c37bb2,42639,1732486469476}] 2024-11-24T22:14:33,709 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=79c2dd6f654fabefe0fc06b8199e8562, regionState=OPENING, regionLocation=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:33,710 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=acdab7e426c93b10b34b4d331b21cef0, regionState=OPENING, regionLocation=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:33,710 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/5160ff1dd0e56b08fbca2ffe581141d4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T22:14:33,711 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=ce1f0bcb3026cb1cbbd9e63c21b83f99, regionState=OPENING, regionLocation=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:33,712 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=967acf99df098c28d8f837a9f0746a90, regionState=OPENING, regionLocation=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:33,711 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 5160ff1dd0e56b08fbca2ffe581141d4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72216867, jitterRate=0.07611517608165741}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T22:14:33,712 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=94f2fa48145fe3bdeffbfded6a4edd73, regionState=OPENING, regionLocation=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:33,712 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5160ff1dd0e56b08fbca2ffe581141d4 2024-11-24T22:14:33,713 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=736f44b8f348e2a5a96dfd3abc5e62b2, regionState=OPENING, regionLocation=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:33,714 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=79c2dd6f654fabefe0fc06b8199e8562, ASSIGN because future has completed 2024-11-24T22:14:33,714 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 79c2dd6f654fabefe0fc06b8199e8562, server=bd6592c37bb2,38125,1732486469405}] 2024-11-24T22:14:33,716 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 5160ff1dd0e56b08fbca2ffe581141d4: Running coprocessor pre-open hook at 1732486473683Writing region info on filesystem at 1732486473683Initializing all the Stores at 1732486473686 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486473687 (+1 ms)Cleaning up temporary data from old regions at 1732486473696 (+9 ms)Running coprocessor post-open hooks at 1732486473713 (+17 ms)Region opened successfully at 1732486473716 (+3 ms) 2024-11-24T22:14:33,717 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=acdab7e426c93b10b34b4d331b21cef0, ASSIGN because future has completed 2024-11-24T22:14:33,720 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for test,nnn,1732486472486.5160ff1dd0e56b08fbca2ffe581141d4., pid=32, masterSystemTime=1732486473664 2024-11-24T22:14:33,720 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ce1f0bcb3026cb1cbbd9e63c21b83f99, ASSIGN because future has completed 2024-11-24T22:14:33,721 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=967acf99df098c28d8f837a9f0746a90, ASSIGN because future has completed 2024-11-24T22:14:33,726 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=94f2fa48145fe3bdeffbfded6a4edd73, ASSIGN because future has completed 2024-11-24T22:14:33,728 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open test,ppp,1732486472486.cca6a66154e5c299fad3a247afbfb5fd. 2024-11-24T22:14:33,728 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => cca6a66154e5c299fad3a247afbfb5fd, NAME => 'test,ppp,1732486472486.cca6a66154e5c299fad3a247afbfb5fd.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-24T22:14:33,729 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test cca6a66154e5c299fad3a247afbfb5fd 2024-11-24T22:14:33,729 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated test,ppp,1732486472486.cca6a66154e5c299fad3a247afbfb5fd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:33,729 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for cca6a66154e5c299fad3a247afbfb5fd 2024-11-24T22:14:33,729 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for cca6a66154e5c299fad3a247afbfb5fd 2024-11-24T22:14:33,730 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=736f44b8f348e2a5a96dfd3abc5e62b2, ASSIGN because future has completed 2024-11-24T22:14:33,731 INFO [StoreOpener-cca6a66154e5c299fad3a247afbfb5fd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region cca6a66154e5c299fad3a247afbfb5fd 2024-11-24T22:14:33,733 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for test,nnn,1732486472486.5160ff1dd0e56b08fbca2ffe581141d4. 2024-11-24T22:14:33,733 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened test,nnn,1732486472486.5160ff1dd0e56b08fbca2ffe581141d4. 2024-11-24T22:14:33,733 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open test,ccc,1732486472486.c68a44a11eeaccf8a3c2f49ba1446c1e. 2024-11-24T22:14:33,733 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => c68a44a11eeaccf8a3c2f49ba1446c1e, NAME => 'test,ccc,1732486472486.c68a44a11eeaccf8a3c2f49ba1446c1e.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-24T22:14:33,733 INFO [StoreOpener-cca6a66154e5c299fad3a247afbfb5fd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cca6a66154e5c299fad3a247afbfb5fd columnFamilyName family 2024-11-24T22:14:33,734 DEBUG [StoreOpener-cca6a66154e5c299fad3a247afbfb5fd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:33,734 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c68a44a11eeaccf8a3c2f49ba1446c1e 2024-11-24T22:14:33,734 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated test,ccc,1732486472486.c68a44a11eeaccf8a3c2f49ba1446c1e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:33,734 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for c68a44a11eeaccf8a3c2f49ba1446c1e 2024-11-24T22:14:33,734 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for c68a44a11eeaccf8a3c2f49ba1446c1e 2024-11-24T22:14:33,735 INFO [StoreOpener-cca6a66154e5c299fad3a247afbfb5fd-1 {}] regionserver.HStore(327): Store=cca6a66154e5c299fad3a247afbfb5fd/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:14:33,735 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for cca6a66154e5c299fad3a247afbfb5fd 2024-11-24T22:14:33,737 INFO [StoreOpener-c68a44a11eeaccf8a3c2f49ba1446c1e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c68a44a11eeaccf8a3c2f49ba1446c1e 2024-11-24T22:14:33,737 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=5160ff1dd0e56b08fbca2ffe581141d4, regionState=OPEN, openSeqNum=2, regionLocation=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:33,737 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/cca6a66154e5c299fad3a247afbfb5fd 2024-11-24T22:14:33,739 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/cca6a66154e5c299fad3a247afbfb5fd 2024-11-24T22:14:33,740 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 736f44b8f348e2a5a96dfd3abc5e62b2, server=bd6592c37bb2,38125,1732486469405}] 2024-11-24T22:14:33,741 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for cca6a66154e5c299fad3a247afbfb5fd 2024-11-24T22:14:33,742 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T22:14:33,741 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for cca6a66154e5c299fad3a247afbfb5fd 2024-11-24T22:14:33,746 INFO [StoreOpener-c68a44a11eeaccf8a3c2f49ba1446c1e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c68a44a11eeaccf8a3c2f49ba1446c1e columnFamilyName family 2024-11-24T22:14:33,746 DEBUG [StoreOpener-c68a44a11eeaccf8a3c2f49ba1446c1e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:33,746 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5160ff1dd0e56b08fbca2ffe581141d4, server=bd6592c37bb2,42639,1732486469476 because future has completed 2024-11-24T22:14:33,747 INFO [StoreOpener-c68a44a11eeaccf8a3c2f49ba1446c1e-1 {}] regionserver.HStore(327): Store=c68a44a11eeaccf8a3c2f49ba1446c1e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:14:33,747 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for c68a44a11eeaccf8a3c2f49ba1446c1e 2024-11-24T22:14:33,747 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for cca6a66154e5c299fad3a247afbfb5fd 2024-11-24T22:14:33,748 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/c68a44a11eeaccf8a3c2f49ba1446c1e 2024-11-24T22:14:33,749 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/c68a44a11eeaccf8a3c2f49ba1446c1e 2024-11-24T22:14:33,750 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for c68a44a11eeaccf8a3c2f49ba1446c1e 2024-11-24T22:14:33,750 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for c68a44a11eeaccf8a3c2f49ba1446c1e 2024-11-24T22:14:33,753 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46527, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T22:14:33,754 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for c68a44a11eeaccf8a3c2f49ba1446c1e 2024-11-24T22:14:33,760 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=18 2024-11-24T22:14:33,760 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure 5160ff1dd0e56b08fbca2ffe581141d4, server=bd6592c37bb2,42639,1732486469476 in 216 msec 2024-11-24T22:14:33,767 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=5160ff1dd0e56b08fbca2ffe581141d4, ASSIGN in 472 msec 2024-11-24T22:14:33,767 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 94f2fa48145fe3bdeffbfded6a4edd73, server=bd6592c37bb2,38125,1732486469405}] 2024-11-24T22:14:33,771 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/cca6a66154e5c299fad3a247afbfb5fd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T22:14:33,772 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened cca6a66154e5c299fad3a247afbfb5fd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72242339, jitterRate=0.0764947384595871}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T22:14:33,772 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cca6a66154e5c299fad3a247afbfb5fd 2024-11-24T22:14:33,772 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for cca6a66154e5c299fad3a247afbfb5fd: Running coprocessor pre-open hook at 1732486473729Writing region info on filesystem at 1732486473729Initializing all the Stores at 1732486473731 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486473731Cleaning up temporary data from old regions at 1732486473742 (+11 ms)Running coprocessor post-open hooks at 1732486473772 (+30 ms)Region opened successfully at 1732486473772 2024-11-24T22:14:33,774 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1732486472486.cca6a66154e5c299fad3a247afbfb5fd., pid=34, masterSystemTime=1732486473721 2024-11-24T22:14:33,775 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/c68a44a11eeaccf8a3c2f49ba1446c1e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T22:14:33,775 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 967acf99df098c28d8f837a9f0746a90, server=bd6592c37bb2,38125,1732486469405}] 2024-11-24T22:14:33,776 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened c68a44a11eeaccf8a3c2f49ba1446c1e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69582770, jitterRate=0.03686407208442688}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T22:14:33,776 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c68a44a11eeaccf8a3c2f49ba1446c1e 2024-11-24T22:14:33,777 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open test,yyy,1732486472486.79d315a497ec3ad891988cb800bae337. 2024-11-24T22:14:33,777 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => 79d315a497ec3ad891988cb800bae337, NAME => 'test,yyy,1732486472486.79d315a497ec3ad891988cb800bae337.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-24T22:14:33,777 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for c68a44a11eeaccf8a3c2f49ba1446c1e: Running coprocessor pre-open hook at 1732486473735Writing region info on filesystem at 1732486473735Initializing all the Stores at 1732486473736 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486473736Cleaning up temporary data from old regions at 1732486473750 (+14 ms)Running coprocessor post-open hooks at 1732486473776 (+26 ms)Region opened successfully at 1732486473777 (+1 ms) 2024-11-24T22:14:33,778 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ppp,1732486472486.cca6a66154e5c299fad3a247afbfb5fd. 2024-11-24T22:14:33,778 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened test,ppp,1732486472486.cca6a66154e5c299fad3a247afbfb5fd. 2024-11-24T22:14:33,778 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open test,ddd,1732486472486.698dd9d64bdaf6a81e797d2590ff86f9. 2024-11-24T22:14:33,778 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 698dd9d64bdaf6a81e797d2590ff86f9, NAME => 'test,ddd,1732486472486.698dd9d64bdaf6a81e797d2590ff86f9.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-11-24T22:14:33,779 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 698dd9d64bdaf6a81e797d2590ff86f9 2024-11-24T22:14:33,779 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated test,ddd,1732486472486.698dd9d64bdaf6a81e797d2590ff86f9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:33,779 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 79d315a497ec3ad891988cb800bae337 2024-11-24T22:14:33,779 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 698dd9d64bdaf6a81e797d2590ff86f9 2024-11-24T22:14:33,779 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 698dd9d64bdaf6a81e797d2590ff86f9 2024-11-24T22:14:33,779 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1732486472486.c68a44a11eeaccf8a3c2f49ba1446c1e., pid=33, masterSystemTime=1732486473664 2024-11-24T22:14:33,780 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated test,yyy,1732486472486.79d315a497ec3ad891988cb800bae337.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:33,780 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for 79d315a497ec3ad891988cb800bae337 2024-11-24T22:14:33,780 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for 79d315a497ec3ad891988cb800bae337 2024-11-24T22:14:33,784 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1732486472486.c68a44a11eeaccf8a3c2f49ba1446c1e. 2024-11-24T22:14:33,784 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened test,ccc,1732486472486.c68a44a11eeaccf8a3c2f49ba1446c1e. 2024-11-24T22:14:33,785 INFO [StoreOpener-698dd9d64bdaf6a81e797d2590ff86f9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 698dd9d64bdaf6a81e797d2590ff86f9 2024-11-24T22:14:33,785 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open test,zzz,1732486472486.ed5b70c013e388180062b1513b980a07. 2024-11-24T22:14:33,785 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => ed5b70c013e388180062b1513b980a07, NAME => 'test,zzz,1732486472486.ed5b70c013e388180062b1513b980a07.', STARTKEY => 'zzz', ENDKEY => ''} 2024-11-24T22:14:33,786 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=c68a44a11eeaccf8a3c2f49ba1446c1e, regionState=OPEN, openSeqNum=2, regionLocation=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:33,787 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ed5b70c013e388180062b1513b980a07 2024-11-24T22:14:33,787 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated test,zzz,1732486472486.ed5b70c013e388180062b1513b980a07.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:33,787 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for ed5b70c013e388180062b1513b980a07 2024-11-24T22:14:33,788 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for ed5b70c013e388180062b1513b980a07 2024-11-24T22:14:33,789 INFO [StoreOpener-698dd9d64bdaf6a81e797d2590ff86f9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 698dd9d64bdaf6a81e797d2590ff86f9 columnFamilyName family 2024-11-24T22:14:33,789 DEBUG [StoreOpener-698dd9d64bdaf6a81e797d2590ff86f9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:33,790 INFO [StoreOpener-698dd9d64bdaf6a81e797d2590ff86f9-1 {}] regionserver.HStore(327): Store=698dd9d64bdaf6a81e797d2590ff86f9/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:14:33,790 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 698dd9d64bdaf6a81e797d2590ff86f9 2024-11-24T22:14:33,790 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure c68a44a11eeaccf8a3c2f49ba1446c1e, server=bd6592c37bb2,42639,1732486469476 because future has completed 2024-11-24T22:14:33,791 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/698dd9d64bdaf6a81e797d2590ff86f9 2024-11-24T22:14:33,791 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=cca6a66154e5c299fad3a247afbfb5fd, regionState=OPEN, openSeqNum=2, regionLocation=bd6592c37bb2,41619,1732486469269 2024-11-24T22:14:33,792 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/698dd9d64bdaf6a81e797d2590ff86f9 2024-11-24T22:14:33,793 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 698dd9d64bdaf6a81e797d2590ff86f9 2024-11-24T22:14:33,793 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 698dd9d64bdaf6a81e797d2590ff86f9 2024-11-24T22:14:33,795 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure cca6a66154e5c299fad3a247afbfb5fd, server=bd6592c37bb2,41619,1732486469269 because future has completed 2024-11-24T22:14:33,796 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 698dd9d64bdaf6a81e797d2590ff86f9 2024-11-24T22:14:33,801 INFO [StoreOpener-79d315a497ec3ad891988cb800bae337-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 79d315a497ec3ad891988cb800bae337 2024-11-24T22:14:33,804 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=20 2024-11-24T22:14:33,804 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure cca6a66154e5c299fad3a247afbfb5fd, server=bd6592c37bb2,41619,1732486469269 in 235 msec 2024-11-24T22:14:33,804 INFO [StoreOpener-79d315a497ec3ad891988cb800bae337-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 79d315a497ec3ad891988cb800bae337 columnFamilyName family 2024-11-24T22:14:33,804 DEBUG [StoreOpener-79d315a497ec3ad891988cb800bae337-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:33,805 INFO [StoreOpener-79d315a497ec3ad891988cb800bae337-1 {}] regionserver.HStore(327): Store=79d315a497ec3ad891988cb800bae337/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:14:33,806 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for 79d315a497ec3ad891988cb800bae337 2024-11-24T22:14:33,808 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/79d315a497ec3ad891988cb800bae337 2024-11-24T22:14:33,808 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=cca6a66154e5c299fad3a247afbfb5fd, ASSIGN in 515 msec 2024-11-24T22:14:33,809 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/79d315a497ec3ad891988cb800bae337 2024-11-24T22:14:33,810 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for 79d315a497ec3ad891988cb800bae337 2024-11-24T22:14:33,810 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for 79d315a497ec3ad891988cb800bae337 2024-11-24T22:14:33,814 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for 79d315a497ec3ad891988cb800bae337 2024-11-24T22:14:33,815 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=7 2024-11-24T22:14:33,815 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure c68a44a11eeaccf8a3c2f49ba1446c1e, server=bd6592c37bb2,42639,1732486469476 in 256 msec 2024-11-24T22:14:33,818 INFO [StoreOpener-ed5b70c013e388180062b1513b980a07-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ed5b70c013e388180062b1513b980a07 2024-11-24T22:14:33,819 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=c68a44a11eeaccf8a3c2f49ba1446c1e, ASSIGN in 527 msec 2024-11-24T22:14:33,820 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure ce1f0bcb3026cb1cbbd9e63c21b83f99, server=bd6592c37bb2,38125,1732486469405}] 2024-11-24T22:14:33,821 INFO [StoreOpener-ed5b70c013e388180062b1513b980a07-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ed5b70c013e388180062b1513b980a07 columnFamilyName family 2024-11-24T22:14:33,821 DEBUG [StoreOpener-ed5b70c013e388180062b1513b980a07-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:33,821 INFO [StoreOpener-ed5b70c013e388180062b1513b980a07-1 {}] regionserver.HStore(327): Store=ed5b70c013e388180062b1513b980a07/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:14:33,822 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for ed5b70c013e388180062b1513b980a07 2024-11-24T22:14:33,823 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/ed5b70c013e388180062b1513b980a07 2024-11-24T22:14:33,824 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/ed5b70c013e388180062b1513b980a07 2024-11-24T22:14:33,826 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for ed5b70c013e388180062b1513b980a07 2024-11-24T22:14:33,827 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for ed5b70c013e388180062b1513b980a07 2024-11-24T22:14:33,827 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/698dd9d64bdaf6a81e797d2590ff86f9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T22:14:33,828 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 698dd9d64bdaf6a81e797d2590ff86f9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67693650, jitterRate=0.008713990449905396}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T22:14:33,828 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 698dd9d64bdaf6a81e797d2590ff86f9 2024-11-24T22:14:33,828 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 698dd9d64bdaf6a81e797d2590ff86f9: Running coprocessor pre-open hook at 1732486473779Writing region info on filesystem at 1732486473779Initializing all the Stores at 1732486473783 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486473783Cleaning up temporary data from old regions at 1732486473793 (+10 ms)Running coprocessor post-open hooks at 1732486473828 (+35 ms)Region opened successfully at 1732486473828 2024-11-24T22:14:33,829 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ddd,1732486472486.698dd9d64bdaf6a81e797d2590ff86f9., pid=37, masterSystemTime=1732486473721 2024-11-24T22:14:33,835 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for ed5b70c013e388180062b1513b980a07 2024-11-24T22:14:33,836 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/79d315a497ec3ad891988cb800bae337/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T22:14:33,836 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened 79d315a497ec3ad891988cb800bae337; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74245745, jitterRate=0.10634781420230865}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T22:14:33,837 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 79d315a497ec3ad891988cb800bae337 2024-11-24T22:14:33,837 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ddd,1732486472486.698dd9d64bdaf6a81e797d2590ff86f9. 2024-11-24T22:14:33,837 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened test,ddd,1732486472486.698dd9d64bdaf6a81e797d2590ff86f9. 2024-11-24T22:14:33,837 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for 79d315a497ec3ad891988cb800bae337: Running coprocessor pre-open hook at 1732486473784Writing region info on filesystem at 1732486473784Initializing all the Stores at 1732486473787 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486473787Cleaning up temporary data from old regions at 1732486473810 (+23 ms)Running coprocessor post-open hooks at 1732486473837 (+27 ms)Region opened successfully at 1732486473837 2024-11-24T22:14:33,837 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(132): Open test,sss,1732486472486.84dd509902edc43083bc78fe4b70922c. 2024-11-24T22:14:33,837 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7752): Opening region: {ENCODED => 84dd509902edc43083bc78fe4b70922c, NAME => 'test,sss,1732486472486.84dd509902edc43083bc78fe4b70922c.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-24T22:14:33,838 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 84dd509902edc43083bc78fe4b70922c 2024-11-24T22:14:33,838 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(898): Instantiated test,sss,1732486472486.84dd509902edc43083bc78fe4b70922c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:33,838 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7794): checking encryption for 84dd509902edc43083bc78fe4b70922c 2024-11-24T22:14:33,838 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7797): checking classloading for 84dd509902edc43083bc78fe4b70922c 2024-11-24T22:14:33,839 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1732486472486.79d315a497ec3ad891988cb800bae337., pid=36, masterSystemTime=1732486473741 2024-11-24T22:14:33,840 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=698dd9d64bdaf6a81e797d2590ff86f9, regionState=OPEN, openSeqNum=2, regionLocation=bd6592c37bb2,41619,1732486469269 2024-11-24T22:14:33,843 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 698dd9d64bdaf6a81e797d2590ff86f9, server=bd6592c37bb2,41619,1732486469269 because future has completed 2024-11-24T22:14:33,845 INFO [StoreOpener-84dd509902edc43083bc78fe4b70922c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 84dd509902edc43083bc78fe4b70922c 2024-11-24T22:14:33,846 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1732486472486.79d315a497ec3ad891988cb800bae337. 2024-11-24T22:14:33,846 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/ed5b70c013e388180062b1513b980a07/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T22:14:33,846 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened test,yyy,1732486472486.79d315a497ec3ad891988cb800bae337. 2024-11-24T22:14:33,846 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open test,uuu,1732486472486.79c2dd6f654fabefe0fc06b8199e8562. 2024-11-24T22:14:33,846 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => 79c2dd6f654fabefe0fc06b8199e8562, NAME => 'test,uuu,1732486472486.79c2dd6f654fabefe0fc06b8199e8562.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-24T22:14:33,847 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened ed5b70c013e388180062b1513b980a07; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68175498, jitterRate=0.0158940851688385}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T22:14:33,847 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ed5b70c013e388180062b1513b980a07 2024-11-24T22:14:33,847 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for ed5b70c013e388180062b1513b980a07: Running coprocessor pre-open hook at 1732486473788Writing region info on filesystem at 1732486473788Initializing all the Stores at 1732486473791 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486473791Cleaning up temporary data from old regions at 1732486473827 (+36 ms)Running coprocessor post-open hooks at 1732486473847 (+20 ms)Region opened successfully at 1732486473847 2024-11-24T22:14:33,848 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 79c2dd6f654fabefe0fc06b8199e8562 2024-11-24T22:14:33,848 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for test,zzz,1732486472486.ed5b70c013e388180062b1513b980a07., pid=31, masterSystemTime=1732486473664 2024-11-24T22:14:33,849 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated test,uuu,1732486472486.79c2dd6f654fabefe0fc06b8199e8562.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:33,849 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for 79c2dd6f654fabefe0fc06b8199e8562 2024-11-24T22:14:33,849 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for 79c2dd6f654fabefe0fc06b8199e8562 2024-11-24T22:14:33,851 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for test,zzz,1732486472486.ed5b70c013e388180062b1513b980a07. 2024-11-24T22:14:33,851 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened test,zzz,1732486472486.ed5b70c013e388180062b1513b980a07. 2024-11-24T22:14:33,851 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open test,hhh,1732486472486.72a41015f7b88194cc3a08feb66aa4f9. 2024-11-24T22:14:33,852 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => 72a41015f7b88194cc3a08feb66aa4f9, NAME => 'test,hhh,1732486472486.72a41015f7b88194cc3a08feb66aa4f9.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-24T22:14:33,852 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 72a41015f7b88194cc3a08feb66aa4f9 2024-11-24T22:14:33,852 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=8 2024-11-24T22:14:33,852 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 698dd9d64bdaf6a81e797d2590ff86f9, server=bd6592c37bb2,41619,1732486469269 in 251 msec 2024-11-24T22:14:33,852 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated test,hhh,1732486472486.72a41015f7b88194cc3a08feb66aa4f9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:33,852 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for 72a41015f7b88194cc3a08feb66aa4f9 2024-11-24T22:14:33,853 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for 72a41015f7b88194cc3a08feb66aa4f9 2024-11-24T22:14:33,855 INFO [StoreOpener-84dd509902edc43083bc78fe4b70922c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 84dd509902edc43083bc78fe4b70922c columnFamilyName family 2024-11-24T22:14:33,855 DEBUG [StoreOpener-84dd509902edc43083bc78fe4b70922c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:33,855 INFO [StoreOpener-84dd509902edc43083bc78fe4b70922c-1 {}] regionserver.HStore(327): Store=84dd509902edc43083bc78fe4b70922c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:14:33,856 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1038): replaying wal for 84dd509902edc43083bc78fe4b70922c 2024-11-24T22:14:33,856 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN in 563 msec 2024-11-24T22:14:33,857 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/84dd509902edc43083bc78fe4b70922c 2024-11-24T22:14:33,858 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/84dd509902edc43083bc78fe4b70922c 2024-11-24T22:14:33,858 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=ed5b70c013e388180062b1513b980a07, regionState=OPEN, openSeqNum=2, regionLocation=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:33,861 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1048): stopping wal replay for 84dd509902edc43083bc78fe4b70922c 2024-11-24T22:14:33,861 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1060): Cleaning up temporary data for 84dd509902edc43083bc78fe4b70922c 2024-11-24T22:14:33,861 INFO [StoreOpener-72a41015f7b88194cc3a08feb66aa4f9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 72a41015f7b88194cc3a08feb66aa4f9 2024-11-24T22:14:33,863 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure ed5b70c013e388180062b1513b980a07, server=bd6592c37bb2,42639,1732486469476 because future has completed 2024-11-24T22:14:33,863 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=79d315a497ec3ad891988cb800bae337, regionState=OPEN, openSeqNum=2, regionLocation=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:33,864 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1093): writing seq id for 84dd509902edc43083bc78fe4b70922c 2024-11-24T22:14:33,865 INFO [StoreOpener-72a41015f7b88194cc3a08feb66aa4f9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 72a41015f7b88194cc3a08feb66aa4f9 columnFamilyName family 2024-11-24T22:14:33,865 DEBUG [StoreOpener-72a41015f7b88194cc3a08feb66aa4f9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:33,859 INFO [StoreOpener-79c2dd6f654fabefe0fc06b8199e8562-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 79c2dd6f654fabefe0fc06b8199e8562 2024-11-24T22:14:33,867 INFO [StoreOpener-72a41015f7b88194cc3a08feb66aa4f9-1 {}] regionserver.HStore(327): Store=72a41015f7b88194cc3a08feb66aa4f9/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:14:33,868 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for 72a41015f7b88194cc3a08feb66aa4f9 2024-11-24T22:14:33,869 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 79d315a497ec3ad891988cb800bae337, server=bd6592c37bb2,38125,1732486469405 because future has completed 2024-11-24T22:14:33,869 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/72a41015f7b88194cc3a08feb66aa4f9 2024-11-24T22:14:33,869 INFO [StoreOpener-79c2dd6f654fabefe0fc06b8199e8562-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 79c2dd6f654fabefe0fc06b8199e8562 columnFamilyName family 2024-11-24T22:14:33,869 DEBUG [StoreOpener-79c2dd6f654fabefe0fc06b8199e8562-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:33,869 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/72a41015f7b88194cc3a08feb66aa4f9 2024-11-24T22:14:33,870 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for 72a41015f7b88194cc3a08feb66aa4f9 2024-11-24T22:14:33,870 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for 72a41015f7b88194cc3a08feb66aa4f9 2024-11-24T22:14:33,870 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36279 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=bd6592c37bb2,38125,1732486469405, table=test, region=79d315a497ec3ad891988cb800bae337. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T22:14:33,873 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for 72a41015f7b88194cc3a08feb66aa4f9 2024-11-24T22:14:33,873 INFO [StoreOpener-79c2dd6f654fabefe0fc06b8199e8562-1 {}] regionserver.HStore(327): Store=79c2dd6f654fabefe0fc06b8199e8562/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:14:33,874 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-24T22:14:33,874 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure ed5b70c013e388180062b1513b980a07, server=bd6592c37bb2,42639,1732486469476 in 364 msec 2024-11-24T22:14:33,874 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for 79c2dd6f654fabefe0fc06b8199e8562 2024-11-24T22:14:33,874 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/84dd509902edc43083bc78fe4b70922c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T22:14:33,876 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/79c2dd6f654fabefe0fc06b8199e8562 2024-11-24T22:14:33,876 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1114): Opened 84dd509902edc43083bc78fe4b70922c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61857673, jitterRate=-0.07824884355068207}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T22:14:33,876 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 84dd509902edc43083bc78fe4b70922c 2024-11-24T22:14:33,877 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1006): Region open journal for 84dd509902edc43083bc78fe4b70922c: Running coprocessor pre-open hook at 1732486473838Writing region info on filesystem at 1732486473838Initializing all the Stores at 1732486473839 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486473839Cleaning up temporary data from old regions at 1732486473861 (+22 ms)Running coprocessor post-open hooks at 1732486473876 (+15 ms)Region opened successfully at 1732486473877 (+1 ms) 2024-11-24T22:14:33,877 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/79c2dd6f654fabefe0fc06b8199e8562 2024-11-24T22:14:33,877 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for 79c2dd6f654fabefe0fc06b8199e8562 2024-11-24T22:14:33,878 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for 79c2dd6f654fabefe0fc06b8199e8562 2024-11-24T22:14:33,878 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1732486472486.84dd509902edc43083bc78fe4b70922c., pid=42, masterSystemTime=1732486473721 2024-11-24T22:14:33,879 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=ed5b70c013e388180062b1513b980a07, ASSIGN in 585 msec 2024-11-24T22:14:33,881 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for 79c2dd6f654fabefe0fc06b8199e8562 2024-11-24T22:14:33,885 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/79c2dd6f654fabefe0fc06b8199e8562/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T22:14:33,887 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1732486472486.84dd509902edc43083bc78fe4b70922c. 2024-11-24T22:14:33,887 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(153): Opened test,sss,1732486472486.84dd509902edc43083bc78fe4b70922c. 2024-11-24T22:14:33,887 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open test,ggg,1732486472486.f72c935213180e3ff989af795022b7e4. 2024-11-24T22:14:33,887 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => f72c935213180e3ff989af795022b7e4, NAME => 'test,ggg,1732486472486.f72c935213180e3ff989af795022b7e4.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-24T22:14:33,887 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=29 2024-11-24T22:14:33,888 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 79d315a497ec3ad891988cb800bae337, server=bd6592c37bb2,38125,1732486469405 in 296 msec 2024-11-24T22:14:33,888 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f72c935213180e3ff989af795022b7e4 2024-11-24T22:14:33,888 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated test,ggg,1732486472486.f72c935213180e3ff989af795022b7e4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:33,888 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for f72c935213180e3ff989af795022b7e4 2024-11-24T22:14:33,888 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for f72c935213180e3ff989af795022b7e4 2024-11-24T22:14:33,892 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=79d315a497ec3ad891988cb800bae337, ASSIGN in 599 msec 2024-11-24T22:14:33,892 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened 79c2dd6f654fabefe0fc06b8199e8562; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60920616, jitterRate=-0.09221208095550537}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T22:14:33,892 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 79c2dd6f654fabefe0fc06b8199e8562 2024-11-24T22:14:33,892 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for 79c2dd6f654fabefe0fc06b8199e8562: Running coprocessor pre-open hook at 1732486473849Writing region info on filesystem at 1732486473850 (+1 ms)Initializing all the Stores at 1732486473852 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486473852Cleaning up temporary data from old regions at 1732486473878 (+26 ms)Running coprocessor post-open hooks at 1732486473892 (+14 ms)Region opened successfully at 1732486473892 2024-11-24T22:14:33,896 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/72a41015f7b88194cc3a08feb66aa4f9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T22:14:33,897 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened 72a41015f7b88194cc3a08feb66aa4f9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60689172, jitterRate=-0.09566086530685425}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T22:14:33,897 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 72a41015f7b88194cc3a08feb66aa4f9 2024-11-24T22:14:33,897 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for 72a41015f7b88194cc3a08feb66aa4f9: Running coprocessor pre-open hook at 1732486473853Writing region info on filesystem at 1732486473853Initializing all the Stores at 1732486473855 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486473855Cleaning up temporary data from old regions at 1732486473870 (+15 ms)Running coprocessor post-open hooks at 1732486473897 (+27 ms)Region opened successfully at 1732486473897 2024-11-24T22:14:33,899 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1732486472486.79c2dd6f654fabefe0fc06b8199e8562., pid=51, masterSystemTime=1732486473741 2024-11-24T22:14:33,899 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1732486472486.72a41015f7b88194cc3a08feb66aa4f9., pid=46, masterSystemTime=1732486473821 2024-11-24T22:14:33,900 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=84dd509902edc43083bc78fe4b70922c, regionState=OPEN, openSeqNum=2, regionLocation=bd6592c37bb2,41619,1732486469269 2024-11-24T22:14:33,905 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,41619,1732486469269 because future has completed 2024-11-24T22:14:33,906 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for test,uuu,1732486472486.79c2dd6f654fabefe0fc06b8199e8562. 2024-11-24T22:14:33,906 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened test,uuu,1732486472486.79c2dd6f654fabefe0fc06b8199e8562. 2024-11-24T22:14:33,906 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open test,bbb,1732486472486.4279880609a62ccb9e9fb29e79bdf7e7. 2024-11-24T22:14:33,906 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => 4279880609a62ccb9e9fb29e79bdf7e7, NAME => 'test,bbb,1732486472486.4279880609a62ccb9e9fb29e79bdf7e7.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-11-24T22:14:33,907 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4279880609a62ccb9e9fb29e79bdf7e7 2024-11-24T22:14:33,907 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated test,bbb,1732486472486.4279880609a62ccb9e9fb29e79bdf7e7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:33,907 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for 4279880609a62ccb9e9fb29e79bdf7e7 2024-11-24T22:14:33,907 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for 4279880609a62ccb9e9fb29e79bdf7e7 2024-11-24T22:14:33,909 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=79c2dd6f654fabefe0fc06b8199e8562, regionState=OPEN, openSeqNum=2, regionLocation=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:33,909 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1732486472486.72a41015f7b88194cc3a08feb66aa4f9. 2024-11-24T22:14:33,909 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened test,hhh,1732486472486.72a41015f7b88194cc3a08feb66aa4f9. 2024-11-24T22:14:33,909 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(132): Open test,www,1732486472486.50d9b38af2d6413e500860133d150aa8. 2024-11-24T22:14:33,910 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7752): Opening region: {ENCODED => 50d9b38af2d6413e500860133d150aa8, NAME => 'test,www,1732486472486.50d9b38af2d6413e500860133d150aa8.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-11-24T22:14:33,910 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 50d9b38af2d6413e500860133d150aa8 2024-11-24T22:14:33,910 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(898): Instantiated test,www,1732486472486.50d9b38af2d6413e500860133d150aa8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:33,910 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7794): checking encryption for 50d9b38af2d6413e500860133d150aa8 2024-11-24T22:14:33,910 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7797): checking classloading for 50d9b38af2d6413e500860133d150aa8 2024-11-24T22:14:33,914 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 79c2dd6f654fabefe0fc06b8199e8562, server=bd6592c37bb2,38125,1732486469405 because future has completed 2024-11-24T22:14:33,918 INFO [StoreOpener-f72c935213180e3ff989af795022b7e4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f72c935213180e3ff989af795022b7e4 2024-11-24T22:14:33,918 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=72a41015f7b88194cc3a08feb66aa4f9, regionState=OPEN, openSeqNum=2, regionLocation=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:33,924 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 72a41015f7b88194cc3a08feb66aa4f9, server=bd6592c37bb2,42639,1732486469476 because future has completed 2024-11-24T22:14:33,924 INFO [StoreOpener-f72c935213180e3ff989af795022b7e4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f72c935213180e3ff989af795022b7e4 columnFamilyName family 2024-11-24T22:14:33,924 DEBUG [StoreOpener-f72c935213180e3ff989af795022b7e4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:33,925 INFO [StoreOpener-f72c935213180e3ff989af795022b7e4-1 {}] regionserver.HStore(327): Store=f72c935213180e3ff989af795022b7e4/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:14:33,926 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for f72c935213180e3ff989af795022b7e4 2024-11-24T22:14:33,927 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/f72c935213180e3ff989af795022b7e4 2024-11-24T22:14:33,928 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/f72c935213180e3ff989af795022b7e4 2024-11-24T22:14:33,928 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for f72c935213180e3ff989af795022b7e4 2024-11-24T22:14:33,928 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for f72c935213180e3ff989af795022b7e4 2024-11-24T22:14:33,929 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=12 2024-11-24T22:14:33,929 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure 72a41015f7b88194cc3a08feb66aa4f9, server=bd6592c37bb2,42639,1732486469476 in 251 msec 2024-11-24T22:14:33,933 INFO [StoreOpener-4279880609a62ccb9e9fb29e79bdf7e7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4279880609a62ccb9e9fb29e79bdf7e7 2024-11-24T22:14:33,934 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for f72c935213180e3ff989af795022b7e4 2024-11-24T22:14:33,935 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=72a41015f7b88194cc3a08feb66aa4f9, ASSIGN in 641 msec 2024-11-24T22:14:33,937 INFO [StoreOpener-50d9b38af2d6413e500860133d150aa8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 50d9b38af2d6413e500860133d150aa8 2024-11-24T22:14:33,939 INFO [StoreOpener-4279880609a62ccb9e9fb29e79bdf7e7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4279880609a62ccb9e9fb29e79bdf7e7 columnFamilyName family 2024-11-24T22:14:33,939 DEBUG [StoreOpener-4279880609a62ccb9e9fb29e79bdf7e7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:33,942 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=25 2024-11-24T22:14:33,942 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure 79c2dd6f654fabefe0fc06b8199e8562, server=bd6592c37bb2,38125,1732486469405 in 223 msec 2024-11-24T22:14:33,944 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=79c2dd6f654fabefe0fc06b8199e8562, ASSIGN in 653 msec 2024-11-24T22:14:33,946 INFO [StoreOpener-4279880609a62ccb9e9fb29e79bdf7e7-1 {}] regionserver.HStore(327): Store=4279880609a62ccb9e9fb29e79bdf7e7/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:14:33,946 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for 4279880609a62ccb9e9fb29e79bdf7e7 2024-11-24T22:14:33,947 INFO [StoreOpener-50d9b38af2d6413e500860133d150aa8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 50d9b38af2d6413e500860133d150aa8 columnFamilyName family 2024-11-24T22:14:33,947 DEBUG [StoreOpener-50d9b38af2d6413e500860133d150aa8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:33,948 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/4279880609a62ccb9e9fb29e79bdf7e7 2024-11-24T22:14:33,948 INFO [StoreOpener-50d9b38af2d6413e500860133d150aa8-1 {}] regionserver.HStore(327): Store=50d9b38af2d6413e500860133d150aa8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:14:33,948 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1038): replaying wal for 50d9b38af2d6413e500860133d150aa8 2024-11-24T22:14:33,950 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/50d9b38af2d6413e500860133d150aa8 2024-11-24T22:14:33,950 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/4279880609a62ccb9e9fb29e79bdf7e7 2024-11-24T22:14:33,950 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/50d9b38af2d6413e500860133d150aa8 2024-11-24T22:14:33,950 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for 4279880609a62ccb9e9fb29e79bdf7e7 2024-11-24T22:14:33,951 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for 4279880609a62ccb9e9fb29e79bdf7e7 2024-11-24T22:14:33,951 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1048): stopping wal replay for 50d9b38af2d6413e500860133d150aa8 2024-11-24T22:14:33,951 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1060): Cleaning up temporary data for 50d9b38af2d6413e500860133d150aa8 2024-11-24T22:14:33,954 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for 4279880609a62ccb9e9fb29e79bdf7e7 2024-11-24T22:14:33,955 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=23 2024-11-24T22:14:33,955 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,41619,1732486469269 in 303 msec 2024-11-24T22:14:33,955 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): writing seq id for 50d9b38af2d6413e500860133d150aa8 2024-11-24T22:14:33,959 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=84dd509902edc43083bc78fe4b70922c, ASSIGN in 666 msec 2024-11-24T22:14:33,960 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure acdab7e426c93b10b34b4d331b21cef0, server=bd6592c37bb2,38125,1732486469405}] 2024-11-24T22:14:33,974 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/f72c935213180e3ff989af795022b7e4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T22:14:33,975 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened f72c935213180e3ff989af795022b7e4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72957001, jitterRate=0.08714403212070465}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T22:14:33,975 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f72c935213180e3ff989af795022b7e4 2024-11-24T22:14:33,975 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for f72c935213180e3ff989af795022b7e4: Running coprocessor pre-open hook at 1732486473888Writing region info on filesystem at 1732486473888Initializing all the Stores at 1732486473900 (+12 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486473900Cleaning up temporary data from old regions at 1732486473929 (+29 ms)Running coprocessor post-open hooks at 1732486473975 (+46 ms)Region opened successfully at 1732486473975 2024-11-24T22:14:33,978 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ggg,1732486472486.f72c935213180e3ff989af795022b7e4., pid=35, masterSystemTime=1732486473721 2024-11-24T22:14:33,981 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/4279880609a62ccb9e9fb29e79bdf7e7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T22:14:33,981 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ggg,1732486472486.f72c935213180e3ff989af795022b7e4. 2024-11-24T22:14:33,982 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened test,ggg,1732486472486.f72c935213180e3ff989af795022b7e4. 2024-11-24T22:14:33,982 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open test,vvv,1732486472486.fc9f96fd4c44240a85e6b2c001e46153. 2024-11-24T22:14:33,982 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => fc9f96fd4c44240a85e6b2c001e46153, NAME => 'test,vvv,1732486472486.fc9f96fd4c44240a85e6b2c001e46153.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-24T22:14:33,982 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened 4279880609a62ccb9e9fb29e79bdf7e7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66897477, jitterRate=-0.0031499117612838745}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T22:14:33,982 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4279880609a62ccb9e9fb29e79bdf7e7 2024-11-24T22:14:33,982 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test fc9f96fd4c44240a85e6b2c001e46153 2024-11-24T22:14:33,983 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated test,vvv,1732486472486.fc9f96fd4c44240a85e6b2c001e46153.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:33,983 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for 4279880609a62ccb9e9fb29e79bdf7e7: Running coprocessor pre-open hook at 1732486473907Writing region info on filesystem at 1732486473907Initializing all the Stores at 1732486473912 (+5 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486473912Cleaning up temporary data from old regions at 1732486473951 (+39 ms)Running coprocessor post-open hooks at 1732486473982 (+31 ms)Region opened successfully at 1732486473983 (+1 ms) 2024-11-24T22:14:33,983 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for fc9f96fd4c44240a85e6b2c001e46153 2024-11-24T22:14:33,983 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for fc9f96fd4c44240a85e6b2c001e46153 2024-11-24T22:14:33,984 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=f72c935213180e3ff989af795022b7e4, regionState=OPEN, openSeqNum=2, regionLocation=bd6592c37bb2,41619,1732486469269 2024-11-24T22:14:33,984 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for test,bbb,1732486472486.4279880609a62ccb9e9fb29e79bdf7e7., pid=40, masterSystemTime=1732486473741 2024-11-24T22:14:33,989 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure f72c935213180e3ff989af795022b7e4, server=bd6592c37bb2,41619,1732486469269 because future has completed 2024-11-24T22:14:34,002 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/50d9b38af2d6413e500860133d150aa8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T22:14:34,002 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for test,bbb,1732486472486.4279880609a62ccb9e9fb29e79bdf7e7. 2024-11-24T22:14:34,002 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened test,bbb,1732486472486.4279880609a62ccb9e9fb29e79bdf7e7. 2024-11-24T22:14:34,002 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open test,iii,1732486472486.94f2fa48145fe3bdeffbfded6a4edd73. 2024-11-24T22:14:34,002 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=4279880609a62ccb9e9fb29e79bdf7e7, regionState=OPEN, openSeqNum=2, regionLocation=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:34,002 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => 94f2fa48145fe3bdeffbfded6a4edd73, NAME => 'test,iii,1732486472486.94f2fa48145fe3bdeffbfded6a4edd73.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-24T22:14:34,003 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 94f2fa48145fe3bdeffbfded6a4edd73 2024-11-24T22:14:34,003 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated test,iii,1732486472486.94f2fa48145fe3bdeffbfded6a4edd73.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:34,003 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for 94f2fa48145fe3bdeffbfded6a4edd73 2024-11-24T22:14:34,003 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1114): Opened 50d9b38af2d6413e500860133d150aa8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69514127, jitterRate=0.035841211676597595}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T22:14:34,003 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for 94f2fa48145fe3bdeffbfded6a4edd73 2024-11-24T22:14:34,003 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 50d9b38af2d6413e500860133d150aa8 2024-11-24T22:14:34,003 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36279 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=bd6592c37bb2,41619,1732486469269, table=test, region=f72c935213180e3ff989af795022b7e4. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T22:14:34,004 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1006): Region open journal for 50d9b38af2d6413e500860133d150aa8: Running coprocessor pre-open hook at 1732486473911Writing region info on filesystem at 1732486473911Initializing all the Stores at 1732486473914 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486473914Cleaning up temporary data from old regions at 1732486473951 (+37 ms)Running coprocessor post-open hooks at 1732486474003 (+52 ms)Region opened successfully at 1732486474003 2024-11-24T22:14:34,005 INFO [StoreOpener-fc9f96fd4c44240a85e6b2c001e46153-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region fc9f96fd4c44240a85e6b2c001e46153 2024-11-24T22:14:34,005 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2236): Post open deploy tasks for test,www,1732486472486.50d9b38af2d6413e500860133d150aa8., pid=50, masterSystemTime=1732486473821 2024-11-24T22:14:34,007 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4279880609a62ccb9e9fb29e79bdf7e7, server=bd6592c37bb2,38125,1732486469405 because future has completed 2024-11-24T22:14:34,009 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2266): Finished post open deploy task for test,www,1732486472486.50d9b38af2d6413e500860133d150aa8. 2024-11-24T22:14:34,009 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(153): Opened test,www,1732486472486.50d9b38af2d6413e500860133d150aa8. 2024-11-24T22:14:34,009 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(132): Open test,rrr,1732486472486.c802d82f160c5269119d007388afb85e. 2024-11-24T22:14:34,009 INFO [StoreOpener-fc9f96fd4c44240a85e6b2c001e46153-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fc9f96fd4c44240a85e6b2c001e46153 columnFamilyName family 2024-11-24T22:14:34,010 DEBUG [StoreOpener-fc9f96fd4c44240a85e6b2c001e46153-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:34,010 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7752): Opening region: {ENCODED => c802d82f160c5269119d007388afb85e, NAME => 'test,rrr,1732486472486.c802d82f160c5269119d007388afb85e.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-24T22:14:34,010 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c802d82f160c5269119d007388afb85e 2024-11-24T22:14:34,010 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(898): Instantiated test,rrr,1732486472486.c802d82f160c5269119d007388afb85e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:34,010 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7794): checking encryption for c802d82f160c5269119d007388afb85e 2024-11-24T22:14:34,010 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7797): checking classloading for c802d82f160c5269119d007388afb85e 2024-11-24T22:14:34,012 INFO [StoreOpener-fc9f96fd4c44240a85e6b2c001e46153-1 {}] regionserver.HStore(327): Store=fc9f96fd4c44240a85e6b2c001e46153/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:14:34,012 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for fc9f96fd4c44240a85e6b2c001e46153 2024-11-24T22:14:34,014 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=11 2024-11-24T22:14:34,014 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure f72c935213180e3ff989af795022b7e4, server=bd6592c37bb2,41619,1732486469269 in 433 msec 2024-11-24T22:14:34,014 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/fc9f96fd4c44240a85e6b2c001e46153 2024-11-24T22:14:34,014 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/fc9f96fd4c44240a85e6b2c001e46153 2024-11-24T22:14:34,015 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for fc9f96fd4c44240a85e6b2c001e46153 2024-11-24T22:14:34,015 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for fc9f96fd4c44240a85e6b2c001e46153 2024-11-24T22:14:34,015 INFO [StoreOpener-94f2fa48145fe3bdeffbfded6a4edd73-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 94f2fa48145fe3bdeffbfded6a4edd73 2024-11-24T22:14:34,017 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f72c935213180e3ff989af795022b7e4, ASSIGN in 725 msec 2024-11-24T22:14:34,018 INFO [StoreOpener-94f2fa48145fe3bdeffbfded6a4edd73-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 94f2fa48145fe3bdeffbfded6a4edd73 columnFamilyName family 2024-11-24T22:14:34,018 DEBUG [StoreOpener-94f2fa48145fe3bdeffbfded6a4edd73-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:34,018 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for fc9f96fd4c44240a85e6b2c001e46153 2024-11-24T22:14:34,019 INFO [StoreOpener-94f2fa48145fe3bdeffbfded6a4edd73-1 {}] regionserver.HStore(327): Store=94f2fa48145fe3bdeffbfded6a4edd73/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:14:34,019 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=50d9b38af2d6413e500860133d150aa8, regionState=OPEN, openSeqNum=2, regionLocation=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:34,019 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for 94f2fa48145fe3bdeffbfded6a4edd73 2024-11-24T22:14:34,021 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/94f2fa48145fe3bdeffbfded6a4edd73 2024-11-24T22:14:34,021 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/94f2fa48145fe3bdeffbfded6a4edd73 2024-11-24T22:14:34,023 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for 94f2fa48145fe3bdeffbfded6a4edd73 2024-11-24T22:14:34,023 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for 94f2fa48145fe3bdeffbfded6a4edd73 2024-11-24T22:14:34,024 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 50d9b38af2d6413e500860133d150aa8, server=bd6592c37bb2,42639,1732486469476 because future has completed 2024-11-24T22:14:34,025 INFO [StoreOpener-c802d82f160c5269119d007388afb85e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c802d82f160c5269119d007388afb85e 2024-11-24T22:14:34,026 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for 94f2fa48145fe3bdeffbfded6a4edd73 2024-11-24T22:14:34,028 INFO [StoreOpener-c802d82f160c5269119d007388afb85e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c802d82f160c5269119d007388afb85e columnFamilyName family 2024-11-24T22:14:34,028 DEBUG [StoreOpener-c802d82f160c5269119d007388afb85e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:34,029 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=6 2024-11-24T22:14:34,030 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure 4279880609a62ccb9e9fb29e79bdf7e7, server=bd6592c37bb2,38125,1732486469405 in 396 msec 2024-11-24T22:14:34,030 INFO [StoreOpener-c802d82f160c5269119d007388afb85e-1 {}] regionserver.HStore(327): Store=c802d82f160c5269119d007388afb85e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:14:34,030 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1038): replaying wal for c802d82f160c5269119d007388afb85e 2024-11-24T22:14:34,032 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/c802d82f160c5269119d007388afb85e 2024-11-24T22:14:34,033 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/c802d82f160c5269119d007388afb85e 2024-11-24T22:14:34,034 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1048): stopping wal replay for c802d82f160c5269119d007388afb85e 2024-11-24T22:14:34,034 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1060): Cleaning up temporary data for c802d82f160c5269119d007388afb85e 2024-11-24T22:14:34,035 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=4279880609a62ccb9e9fb29e79bdf7e7, ASSIGN in 741 msec 2024-11-24T22:14:34,037 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): writing seq id for c802d82f160c5269119d007388afb85e 2024-11-24T22:14:34,041 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=27 2024-11-24T22:14:34,041 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure 50d9b38af2d6413e500860133d150aa8, server=bd6592c37bb2,42639,1732486469476 in 335 msec 2024-11-24T22:14:34,046 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=50d9b38af2d6413e500860133d150aa8, ASSIGN in 752 msec 2024-11-24T22:14:34,050 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/fc9f96fd4c44240a85e6b2c001e46153/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T22:14:34,051 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened fc9f96fd4c44240a85e6b2c001e46153; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61483304, jitterRate=-0.08382737636566162}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T22:14:34,051 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fc9f96fd4c44240a85e6b2c001e46153 2024-11-24T22:14:34,051 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for fc9f96fd4c44240a85e6b2c001e46153: Running coprocessor pre-open hook at 1732486473983Writing region info on filesystem at 1732486473983Initializing all the Stores at 1732486473985 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486473985Cleaning up temporary data from old regions at 1732486474015 (+30 ms)Running coprocessor post-open hooks at 1732486474051 (+36 ms)Region opened successfully at 1732486474051 2024-11-24T22:14:34,052 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1732486472486.fc9f96fd4c44240a85e6b2c001e46153., pid=43, masterSystemTime=1732486473721 2024-11-24T22:14:34,056 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1732486472486.fc9f96fd4c44240a85e6b2c001e46153. 2024-11-24T22:14:34,056 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened test,vvv,1732486472486.fc9f96fd4c44240a85e6b2c001e46153. 2024-11-24T22:14:34,056 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open test,jjj,1732486472486.9b2e9ce57ffd0cd61c72602e5f6d9cae. 2024-11-24T22:14:34,056 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => 9b2e9ce57ffd0cd61c72602e5f6d9cae, NAME => 'test,jjj,1732486472486.9b2e9ce57ffd0cd61c72602e5f6d9cae.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-24T22:14:34,057 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 9b2e9ce57ffd0cd61c72602e5f6d9cae 2024-11-24T22:14:34,057 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated test,jjj,1732486472486.9b2e9ce57ffd0cd61c72602e5f6d9cae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:34,057 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for 9b2e9ce57ffd0cd61c72602e5f6d9cae 2024-11-24T22:14:34,057 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for 9b2e9ce57ffd0cd61c72602e5f6d9cae 2024-11-24T22:14:34,059 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=fc9f96fd4c44240a85e6b2c001e46153, regionState=OPEN, openSeqNum=2, regionLocation=bd6592c37bb2,41619,1732486469269 2024-11-24T22:14:34,062 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure fc9f96fd4c44240a85e6b2c001e46153, server=bd6592c37bb2,41619,1732486469269 because future has completed 2024-11-24T22:14:34,063 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/94f2fa48145fe3bdeffbfded6a4edd73/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T22:14:34,064 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened 94f2fa48145fe3bdeffbfded6a4edd73; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73213894, jitterRate=0.09097203612327576}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T22:14:34,065 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 94f2fa48145fe3bdeffbfded6a4edd73 2024-11-24T22:14:34,065 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for 94f2fa48145fe3bdeffbfded6a4edd73: Running coprocessor pre-open hook at 1732486474003Writing region info on filesystem at 1732486474003Initializing all the Stores at 1732486474007 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486474007Cleaning up temporary data from old regions at 1732486474023 (+16 ms)Running coprocessor post-open hooks at 1732486474065 (+42 ms)Region opened successfully at 1732486474065 2024-11-24T22:14:34,066 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1732486472486.94f2fa48145fe3bdeffbfded6a4edd73., pid=53, masterSystemTime=1732486473897 2024-11-24T22:14:34,067 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/c802d82f160c5269119d007388afb85e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T22:14:34,068 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=26 2024-11-24T22:14:34,068 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure fc9f96fd4c44240a85e6b2c001e46153, server=bd6592c37bb2,41619,1732486469269 in 414 msec 2024-11-24T22:14:34,068 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1114): Opened c802d82f160c5269119d007388afb85e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74385458, jitterRate=0.10842970013618469}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T22:14:34,068 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c802d82f160c5269119d007388afb85e 2024-11-24T22:14:34,068 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1006): Region open journal for c802d82f160c5269119d007388afb85e: Running coprocessor pre-open hook at 1732486474011Writing region info on filesystem at 1732486474011Initializing all the Stores at 1732486474013 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486474013Cleaning up temporary data from old regions at 1732486474034 (+21 ms)Running coprocessor post-open hooks at 1732486474068 (+34 ms)Region opened successfully at 1732486474068 2024-11-24T22:14:34,069 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1732486472486.94f2fa48145fe3bdeffbfded6a4edd73. 2024-11-24T22:14:34,069 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened test,iii,1732486472486.94f2fa48145fe3bdeffbfded6a4edd73. 2024-11-24T22:14:34,070 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open test,ooo,1732486472486.ce1f0bcb3026cb1cbbd9e63c21b83f99. 2024-11-24T22:14:34,070 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => ce1f0bcb3026cb1cbbd9e63c21b83f99, NAME => 'test,ooo,1732486472486.ce1f0bcb3026cb1cbbd9e63c21b83f99.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-11-24T22:14:34,070 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1732486472486.c802d82f160c5269119d007388afb85e., pid=49, masterSystemTime=1732486473821 2024-11-24T22:14:34,071 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ce1f0bcb3026cb1cbbd9e63c21b83f99 2024-11-24T22:14:34,071 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated test,ooo,1732486472486.ce1f0bcb3026cb1cbbd9e63c21b83f99.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:34,071 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for ce1f0bcb3026cb1cbbd9e63c21b83f99 2024-11-24T22:14:34,071 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for ce1f0bcb3026cb1cbbd9e63c21b83f99 2024-11-24T22:14:34,072 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN in 779 msec 2024-11-24T22:14:34,074 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=94f2fa48145fe3bdeffbfded6a4edd73, regionState=OPEN, openSeqNum=2, regionLocation=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:34,074 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1732486472486.c802d82f160c5269119d007388afb85e. 2024-11-24T22:14:34,074 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(153): Opened test,rrr,1732486472486.c802d82f160c5269119d007388afb85e. 2024-11-24T22:14:34,074 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open test,kkk,1732486472486.0b86957c7b81466a818bbff45fac751f. 2024-11-24T22:14:34,074 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => 0b86957c7b81466a818bbff45fac751f, NAME => 'test,kkk,1732486472486.0b86957c7b81466a818bbff45fac751f.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-11-24T22:14:34,075 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 0b86957c7b81466a818bbff45fac751f 2024-11-24T22:14:34,075 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated test,kkk,1732486472486.0b86957c7b81466a818bbff45fac751f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:34,075 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for 0b86957c7b81466a818bbff45fac751f 2024-11-24T22:14:34,075 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for 0b86957c7b81466a818bbff45fac751f 2024-11-24T22:14:34,076 INFO [StoreOpener-9b2e9ce57ffd0cd61c72602e5f6d9cae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 9b2e9ce57ffd0cd61c72602e5f6d9cae 2024-11-24T22:14:34,077 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=c802d82f160c5269119d007388afb85e, regionState=OPEN, openSeqNum=2, regionLocation=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:34,077 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 94f2fa48145fe3bdeffbfded6a4edd73, server=bd6592c37bb2,38125,1732486469405 because future has completed 2024-11-24T22:14:34,079 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure c802d82f160c5269119d007388afb85e, server=bd6592c37bb2,42639,1732486469476 because future has completed 2024-11-24T22:14:34,080 INFO [StoreOpener-9b2e9ce57ffd0cd61c72602e5f6d9cae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9b2e9ce57ffd0cd61c72602e5f6d9cae columnFamilyName family 2024-11-24T22:14:34,080 DEBUG [StoreOpener-9b2e9ce57ffd0cd61c72602e5f6d9cae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:34,081 INFO [StoreOpener-9b2e9ce57ffd0cd61c72602e5f6d9cae-1 {}] regionserver.HStore(327): Store=9b2e9ce57ffd0cd61c72602e5f6d9cae/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:14:34,081 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for 9b2e9ce57ffd0cd61c72602e5f6d9cae 2024-11-24T22:14:34,082 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/9b2e9ce57ffd0cd61c72602e5f6d9cae 2024-11-24T22:14:34,083 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/9b2e9ce57ffd0cd61c72602e5f6d9cae 2024-11-24T22:14:34,083 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for 9b2e9ce57ffd0cd61c72602e5f6d9cae 2024-11-24T22:14:34,084 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for 9b2e9ce57ffd0cd61c72602e5f6d9cae 2024-11-24T22:14:34,086 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=13 2024-11-24T22:14:34,086 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure 94f2fa48145fe3bdeffbfded6a4edd73, server=bd6592c37bb2,38125,1732486469405 in 315 msec 2024-11-24T22:14:34,086 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for 9b2e9ce57ffd0cd61c72602e5f6d9cae 2024-11-24T22:14:34,089 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=94f2fa48145fe3bdeffbfded6a4edd73, ASSIGN in 797 msec 2024-11-24T22:14:34,100 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=22 2024-11-24T22:14:34,101 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure c802d82f160c5269119d007388afb85e, server=bd6592c37bb2,42639,1732486469476 in 399 msec 2024-11-24T22:14:34,103 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=c802d82f160c5269119d007388afb85e, ASSIGN in 812 msec 2024-11-24T22:14:34,104 INFO [StoreOpener-0b86957c7b81466a818bbff45fac751f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 0b86957c7b81466a818bbff45fac751f 2024-11-24T22:14:34,109 INFO [StoreOpener-0b86957c7b81466a818bbff45fac751f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0b86957c7b81466a818bbff45fac751f columnFamilyName family 2024-11-24T22:14:34,109 DEBUG [StoreOpener-0b86957c7b81466a818bbff45fac751f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:34,110 INFO [StoreOpener-0b86957c7b81466a818bbff45fac751f-1 {}] regionserver.HStore(327): Store=0b86957c7b81466a818bbff45fac751f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:14:34,110 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for 0b86957c7b81466a818bbff45fac751f 2024-11-24T22:14:34,111 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/0b86957c7b81466a818bbff45fac751f 2024-11-24T22:14:34,113 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/0b86957c7b81466a818bbff45fac751f 2024-11-24T22:14:34,116 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for 0b86957c7b81466a818bbff45fac751f 2024-11-24T22:14:34,116 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for 0b86957c7b81466a818bbff45fac751f 2024-11-24T22:14:34,119 INFO [StoreOpener-ce1f0bcb3026cb1cbbd9e63c21b83f99-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ce1f0bcb3026cb1cbbd9e63c21b83f99 2024-11-24T22:14:34,121 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for 0b86957c7b81466a818bbff45fac751f 2024-11-24T22:14:34,123 INFO [StoreOpener-ce1f0bcb3026cb1cbbd9e63c21b83f99-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ce1f0bcb3026cb1cbbd9e63c21b83f99 columnFamilyName family 2024-11-24T22:14:34,123 DEBUG [StoreOpener-ce1f0bcb3026cb1cbbd9e63c21b83f99-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:34,126 INFO [StoreOpener-ce1f0bcb3026cb1cbbd9e63c21b83f99-1 {}] regionserver.HStore(327): Store=ce1f0bcb3026cb1cbbd9e63c21b83f99/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:14:34,126 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for ce1f0bcb3026cb1cbbd9e63c21b83f99 2024-11-24T22:14:34,127 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/ce1f0bcb3026cb1cbbd9e63c21b83f99 2024-11-24T22:14:34,128 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/ce1f0bcb3026cb1cbbd9e63c21b83f99 2024-11-24T22:14:34,129 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for ce1f0bcb3026cb1cbbd9e63c21b83f99 2024-11-24T22:14:34,129 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for ce1f0bcb3026cb1cbbd9e63c21b83f99 2024-11-24T22:14:34,131 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/9b2e9ce57ffd0cd61c72602e5f6d9cae/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T22:14:34,132 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened 9b2e9ce57ffd0cd61c72602e5f6d9cae; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68763153, jitterRate=0.024650827050209045}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T22:14:34,132 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9b2e9ce57ffd0cd61c72602e5f6d9cae 2024-11-24T22:14:34,132 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for 9b2e9ce57ffd0cd61c72602e5f6d9cae: Running coprocessor pre-open hook at 1732486474057Writing region info on filesystem at 1732486474057Initializing all the Stores at 1732486474059 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486474059Cleaning up temporary data from old regions at 1732486474084 (+25 ms)Running coprocessor post-open hooks at 1732486474132 (+48 ms)Region opened successfully at 1732486474132 2024-11-24T22:14:34,132 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for ce1f0bcb3026cb1cbbd9e63c21b83f99 2024-11-24T22:14:34,134 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for test,jjj,1732486472486.9b2e9ce57ffd0cd61c72602e5f6d9cae., pid=38, masterSystemTime=1732486473721 2024-11-24T22:14:34,138 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/0b86957c7b81466a818bbff45fac751f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T22:14:34,138 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for test,jjj,1732486472486.9b2e9ce57ffd0cd61c72602e5f6d9cae. 2024-11-24T22:14:34,138 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened test,jjj,1732486472486.9b2e9ce57ffd0cd61c72602e5f6d9cae. 2024-11-24T22:14:34,139 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open test,,1732486472486.4e91326d9778777879bc895ecb241af8. 2024-11-24T22:14:34,139 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=9b2e9ce57ffd0cd61c72602e5f6d9cae, regionState=OPEN, openSeqNum=2, regionLocation=bd6592c37bb2,41619,1732486469269 2024-11-24T22:14:34,139 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => 4e91326d9778777879bc895ecb241af8, NAME => 'test,,1732486472486.4e91326d9778777879bc895ecb241af8.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-24T22:14:34,139 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4e91326d9778777879bc895ecb241af8 2024-11-24T22:14:34,139 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated test,,1732486472486.4e91326d9778777879bc895ecb241af8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:34,140 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for 4e91326d9778777879bc895ecb241af8 2024-11-24T22:14:34,140 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for 4e91326d9778777879bc895ecb241af8 2024-11-24T22:14:34,140 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened 0b86957c7b81466a818bbff45fac751f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72978172, jitterRate=0.0874595046043396}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T22:14:34,140 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0b86957c7b81466a818bbff45fac751f 2024-11-24T22:14:34,141 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for 0b86957c7b81466a818bbff45fac751f: Running coprocessor pre-open hook at 1732486474075Writing region info on filesystem at 1732486474075Initializing all the Stores at 1732486474076 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486474077 (+1 ms)Cleaning up temporary data from old regions at 1732486474116 (+39 ms)Running coprocessor post-open hooks at 1732486474140 (+24 ms)Region opened successfully at 1732486474141 (+1 ms) 2024-11-24T22:14:34,142 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1732486472486.0b86957c7b81466a818bbff45fac751f., pid=47, masterSystemTime=1732486473821 2024-11-24T22:14:34,145 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9b2e9ce57ffd0cd61c72602e5f6d9cae, server=bd6592c37bb2,41619,1732486469269 because future has completed 2024-11-24T22:14:34,146 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for test,kkk,1732486472486.0b86957c7b81466a818bbff45fac751f. 2024-11-24T22:14:34,147 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened test,kkk,1732486472486.0b86957c7b81466a818bbff45fac751f. 2024-11-24T22:14:34,147 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open test,eee,1732486472486.46b79f5a3db63445091682b5e72f635a. 2024-11-24T22:14:34,147 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => 46b79f5a3db63445091682b5e72f635a, NAME => 'test,eee,1732486472486.46b79f5a3db63445091682b5e72f635a.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-24T22:14:34,147 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 46b79f5a3db63445091682b5e72f635a 2024-11-24T22:14:34,148 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated test,eee,1732486472486.46b79f5a3db63445091682b5e72f635a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:34,148 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for 46b79f5a3db63445091682b5e72f635a 2024-11-24T22:14:34,148 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for 46b79f5a3db63445091682b5e72f635a 2024-11-24T22:14:34,153 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=14 2024-11-24T22:14:34,154 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 9b2e9ce57ffd0cd61c72602e5f6d9cae, server=bd6592c37bb2,41619,1732486469269 in 544 msec 2024-11-24T22:14:34,156 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae, ASSIGN in 865 msec 2024-11-24T22:14:34,157 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/ce1f0bcb3026cb1cbbd9e63c21b83f99/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T22:14:34,158 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=0b86957c7b81466a818bbff45fac751f, regionState=OPEN, openSeqNum=2, regionLocation=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:34,158 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened ce1f0bcb3026cb1cbbd9e63c21b83f99; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59987254, jitterRate=-0.10612025856971741}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T22:14:34,158 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ce1f0bcb3026cb1cbbd9e63c21b83f99 2024-11-24T22:14:34,159 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for ce1f0bcb3026cb1cbbd9e63c21b83f99: Running coprocessor pre-open hook at 1732486474072Writing region info on filesystem at 1732486474072Initializing all the Stores at 1732486474073 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486474073Cleaning up temporary data from old regions at 1732486474129 (+56 ms)Running coprocessor post-open hooks at 1732486474158 (+29 ms)Region opened successfully at 1732486474159 (+1 ms) 2024-11-24T22:14:34,161 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1732486472486.ce1f0bcb3026cb1cbbd9e63c21b83f99., pid=55, masterSystemTime=1732486473897 2024-11-24T22:14:34,162 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0b86957c7b81466a818bbff45fac751f, server=bd6592c37bb2,42639,1732486469476 because future has completed 2024-11-24T22:14:34,164 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ooo,1732486472486.ce1f0bcb3026cb1cbbd9e63c21b83f99. 2024-11-24T22:14:34,164 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened test,ooo,1732486472486.ce1f0bcb3026cb1cbbd9e63c21b83f99. 2024-11-24T22:14:34,164 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(132): Open test,fff,1732486472486.736f44b8f348e2a5a96dfd3abc5e62b2. 2024-11-24T22:14:34,164 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7752): Opening region: {ENCODED => 736f44b8f348e2a5a96dfd3abc5e62b2, NAME => 'test,fff,1732486472486.736f44b8f348e2a5a96dfd3abc5e62b2.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-24T22:14:34,165 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 736f44b8f348e2a5a96dfd3abc5e62b2 2024-11-24T22:14:34,165 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(898): Instantiated test,fff,1732486472486.736f44b8f348e2a5a96dfd3abc5e62b2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:34,165 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7794): checking encryption for 736f44b8f348e2a5a96dfd3abc5e62b2 2024-11-24T22:14:34,165 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7797): checking classloading for 736f44b8f348e2a5a96dfd3abc5e62b2 2024-11-24T22:14:34,168 INFO [StoreOpener-4e91326d9778777879bc895ecb241af8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4e91326d9778777879bc895ecb241af8 2024-11-24T22:14:34,169 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=15 2024-11-24T22:14:34,169 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure 0b86957c7b81466a818bbff45fac751f, server=bd6592c37bb2,42639,1732486469476 in 483 msec 2024-11-24T22:14:34,170 INFO [StoreOpener-4e91326d9778777879bc895ecb241af8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4e91326d9778777879bc895ecb241af8 columnFamilyName family 2024-11-24T22:14:34,170 DEBUG [StoreOpener-4e91326d9778777879bc895ecb241af8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:34,171 INFO [StoreOpener-4e91326d9778777879bc895ecb241af8-1 {}] regionserver.HStore(327): Store=4e91326d9778777879bc895ecb241af8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:14:34,171 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for 4e91326d9778777879bc895ecb241af8 2024-11-24T22:14:34,172 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=0b86957c7b81466a818bbff45fac751f, ASSIGN in 880 msec 2024-11-24T22:14:34,172 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/4e91326d9778777879bc895ecb241af8 2024-11-24T22:14:34,173 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/4e91326d9778777879bc895ecb241af8 2024-11-24T22:14:34,173 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=ce1f0bcb3026cb1cbbd9e63c21b83f99, regionState=OPEN, openSeqNum=2, regionLocation=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:34,174 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for 4e91326d9778777879bc895ecb241af8 2024-11-24T22:14:34,174 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for 4e91326d9778777879bc895ecb241af8 2024-11-24T22:14:34,177 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for 4e91326d9778777879bc895ecb241af8 2024-11-24T22:14:34,177 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure ce1f0bcb3026cb1cbbd9e63c21b83f99, server=bd6592c37bb2,38125,1732486469405 because future has completed 2024-11-24T22:14:34,186 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=19 2024-11-24T22:14:34,186 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=19, state=SUCCESS, hasLock=false; OpenRegionProcedure ce1f0bcb3026cb1cbbd9e63c21b83f99, server=bd6592c37bb2,38125,1732486469405 in 363 msec 2024-11-24T22:14:34,188 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=ce1f0bcb3026cb1cbbd9e63c21b83f99, ASSIGN in 897 msec 2024-11-24T22:14:34,188 INFO [StoreOpener-46b79f5a3db63445091682b5e72f635a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 46b79f5a3db63445091682b5e72f635a 2024-11-24T22:14:34,192 INFO [StoreOpener-46b79f5a3db63445091682b5e72f635a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 46b79f5a3db63445091682b5e72f635a columnFamilyName family 2024-11-24T22:14:34,192 DEBUG [StoreOpener-46b79f5a3db63445091682b5e72f635a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:34,192 INFO [StoreOpener-736f44b8f348e2a5a96dfd3abc5e62b2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 736f44b8f348e2a5a96dfd3abc5e62b2 2024-11-24T22:14:34,193 INFO [StoreOpener-46b79f5a3db63445091682b5e72f635a-1 {}] regionserver.HStore(327): Store=46b79f5a3db63445091682b5e72f635a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:14:34,194 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for 46b79f5a3db63445091682b5e72f635a 2024-11-24T22:14:34,195 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/46b79f5a3db63445091682b5e72f635a 2024-11-24T22:14:34,195 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/46b79f5a3db63445091682b5e72f635a 2024-11-24T22:14:34,195 INFO [StoreOpener-736f44b8f348e2a5a96dfd3abc5e62b2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 736f44b8f348e2a5a96dfd3abc5e62b2 columnFamilyName family 2024-11-24T22:14:34,196 DEBUG [StoreOpener-736f44b8f348e2a5a96dfd3abc5e62b2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:34,196 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for 46b79f5a3db63445091682b5e72f635a 2024-11-24T22:14:34,196 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for 46b79f5a3db63445091682b5e72f635a 2024-11-24T22:14:34,199 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for 46b79f5a3db63445091682b5e72f635a 2024-11-24T22:14:34,200 INFO [StoreOpener-736f44b8f348e2a5a96dfd3abc5e62b2-1 {}] regionserver.HStore(327): Store=736f44b8f348e2a5a96dfd3abc5e62b2/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:14:34,200 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1038): replaying wal for 736f44b8f348e2a5a96dfd3abc5e62b2 2024-11-24T22:14:34,202 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/736f44b8f348e2a5a96dfd3abc5e62b2 2024-11-24T22:14:34,202 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/736f44b8f348e2a5a96dfd3abc5e62b2 2024-11-24T22:14:34,205 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/4e91326d9778777879bc895ecb241af8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T22:14:34,206 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened 4e91326d9778777879bc895ecb241af8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64745836, jitterRate=-0.03521186113357544}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T22:14:34,206 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4e91326d9778777879bc895ecb241af8 2024-11-24T22:14:34,206 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for 4e91326d9778777879bc895ecb241af8: Running coprocessor pre-open hook at 1732486474140Writing region info on filesystem at 1732486474140Initializing all the Stores at 1732486474142 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486474142Cleaning up temporary data from old regions at 1732486474174 (+32 ms)Running coprocessor post-open hooks at 1732486474206 (+32 ms)Region opened successfully at 1732486474206 2024-11-24T22:14:34,206 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1048): stopping wal replay for 736f44b8f348e2a5a96dfd3abc5e62b2 2024-11-24T22:14:34,207 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1060): Cleaning up temporary data for 736f44b8f348e2a5a96dfd3abc5e62b2 2024-11-24T22:14:34,208 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1732486472486.4e91326d9778777879bc895ecb241af8., pid=41, masterSystemTime=1732486473721 2024-11-24T22:14:34,212 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for test,,1732486472486.4e91326d9778777879bc895ecb241af8. 2024-11-24T22:14:34,212 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened test,,1732486472486.4e91326d9778777879bc895ecb241af8. 2024-11-24T22:14:34,212 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open test,mmm,1732486472486.dc147c653b22684b8e41092cd6a0c8eb. 2024-11-24T22:14:34,213 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => dc147c653b22684b8e41092cd6a0c8eb, NAME => 'test,mmm,1732486472486.dc147c653b22684b8e41092cd6a0c8eb.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-24T22:14:34,213 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): writing seq id for 736f44b8f348e2a5a96dfd3abc5e62b2 2024-11-24T22:14:34,213 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test dc147c653b22684b8e41092cd6a0c8eb 2024-11-24T22:14:34,213 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated test,mmm,1732486472486.dc147c653b22684b8e41092cd6a0c8eb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:34,213 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for dc147c653b22684b8e41092cd6a0c8eb 2024-11-24T22:14:34,213 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for dc147c653b22684b8e41092cd6a0c8eb 2024-11-24T22:14:34,214 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=4e91326d9778777879bc895ecb241af8, regionState=OPEN, openSeqNum=2, regionLocation=bd6592c37bb2,41619,1732486469269 2024-11-24T22:14:34,219 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e91326d9778777879bc895ecb241af8, server=bd6592c37bb2,41619,1732486469269 because future has completed 2024-11-24T22:14:34,224 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/46b79f5a3db63445091682b5e72f635a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T22:14:34,225 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=5 2024-11-24T22:14:34,225 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 4e91326d9778777879bc895ecb241af8, server=bd6592c37bb2,41619,1732486469269 in 587 msec 2024-11-24T22:14:34,225 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened 46b79f5a3db63445091682b5e72f635a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62759933, jitterRate=-0.06480412185192108}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T22:14:34,225 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 46b79f5a3db63445091682b5e72f635a 2024-11-24T22:14:34,226 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for 46b79f5a3db63445091682b5e72f635a: Running coprocessor pre-open hook at 1732486474148Writing region info on filesystem at 1732486474148Initializing all the Stores at 1732486474150 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486474150Cleaning up temporary data from old regions at 1732486474196 (+46 ms)Running coprocessor post-open hooks at 1732486474226 (+30 ms)Region opened successfully at 1732486474226 2024-11-24T22:14:34,226 INFO [StoreOpener-dc147c653b22684b8e41092cd6a0c8eb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region dc147c653b22684b8e41092cd6a0c8eb 2024-11-24T22:14:34,227 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1732486472486.46b79f5a3db63445091682b5e72f635a., pid=45, masterSystemTime=1732486473821 2024-11-24T22:14:34,230 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=4e91326d9778777879bc895ecb241af8, ASSIGN in 936 msec 2024-11-24T22:14:34,231 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1732486472486.46b79f5a3db63445091682b5e72f635a. 2024-11-24T22:14:34,231 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened test,eee,1732486472486.46b79f5a3db63445091682b5e72f635a. 2024-11-24T22:14:34,231 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(132): Open test,qqq,1732486472486.83e58b0e0989e11c83283bf7bd41306c. 2024-11-24T22:14:34,231 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7752): Opening region: {ENCODED => 83e58b0e0989e11c83283bf7bd41306c, NAME => 'test,qqq,1732486472486.83e58b0e0989e11c83283bf7bd41306c.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-11-24T22:14:34,232 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 83e58b0e0989e11c83283bf7bd41306c 2024-11-24T22:14:34,232 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(898): Instantiated test,qqq,1732486472486.83e58b0e0989e11c83283bf7bd41306c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:34,232 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7794): checking encryption for 83e58b0e0989e11c83283bf7bd41306c 2024-11-24T22:14:34,232 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7797): checking classloading for 83e58b0e0989e11c83283bf7bd41306c 2024-11-24T22:14:34,234 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=46b79f5a3db63445091682b5e72f635a, regionState=OPEN, openSeqNum=2, regionLocation=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:34,237 INFO [StoreOpener-dc147c653b22684b8e41092cd6a0c8eb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dc147c653b22684b8e41092cd6a0c8eb columnFamilyName family 2024-11-24T22:14:34,237 DEBUG [StoreOpener-dc147c653b22684b8e41092cd6a0c8eb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:34,238 INFO [StoreOpener-dc147c653b22684b8e41092cd6a0c8eb-1 {}] regionserver.HStore(327): Store=dc147c653b22684b8e41092cd6a0c8eb/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:14:34,238 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for dc147c653b22684b8e41092cd6a0c8eb 2024-11-24T22:14:34,238 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 46b79f5a3db63445091682b5e72f635a, server=bd6592c37bb2,42639,1732486469476 because future has completed 2024-11-24T22:14:34,240 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/dc147c653b22684b8e41092cd6a0c8eb 2024-11-24T22:14:34,241 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/dc147c653b22684b8e41092cd6a0c8eb 2024-11-24T22:14:34,241 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/736f44b8f348e2a5a96dfd3abc5e62b2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T22:14:34,242 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for dc147c653b22684b8e41092cd6a0c8eb 2024-11-24T22:14:34,242 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for dc147c653b22684b8e41092cd6a0c8eb 2024-11-24T22:14:34,242 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1114): Opened 736f44b8f348e2a5a96dfd3abc5e62b2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70249127, jitterRate=0.04679356515407562}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T22:14:34,242 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 736f44b8f348e2a5a96dfd3abc5e62b2 2024-11-24T22:14:34,242 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1006): Region open journal for 736f44b8f348e2a5a96dfd3abc5e62b2: Running coprocessor pre-open hook at 1732486474165Writing region info on filesystem at 1732486474165Initializing all the Stores at 1732486474167 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486474167Cleaning up temporary data from old regions at 1732486474207 (+40 ms)Running coprocessor post-open hooks at 1732486474242 (+35 ms)Region opened successfully at 1732486474242 2024-11-24T22:14:34,244 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1732486472486.736f44b8f348e2a5a96dfd3abc5e62b2., pid=52, masterSystemTime=1732486473897 2024-11-24T22:14:34,246 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for dc147c653b22684b8e41092cd6a0c8eb 2024-11-24T22:14:34,250 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1732486472486.736f44b8f348e2a5a96dfd3abc5e62b2. 2024-11-24T22:14:34,250 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(153): Opened test,fff,1732486472486.736f44b8f348e2a5a96dfd3abc5e62b2. 2024-11-24T22:14:34,250 INFO [StoreOpener-83e58b0e0989e11c83283bf7bd41306c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 83e58b0e0989e11c83283bf7bd41306c 2024-11-24T22:14:34,250 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(132): Open test,lll,1732486472486.967acf99df098c28d8f837a9f0746a90. 2024-11-24T22:14:34,251 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7752): Opening region: {ENCODED => 967acf99df098c28d8f837a9f0746a90, NAME => 'test,lll,1732486472486.967acf99df098c28d8f837a9f0746a90.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-24T22:14:34,251 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 967acf99df098c28d8f837a9f0746a90 2024-11-24T22:14:34,251 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(898): Instantiated test,lll,1732486472486.967acf99df098c28d8f837a9f0746a90.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:34,251 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7794): checking encryption for 967acf99df098c28d8f837a9f0746a90 2024-11-24T22:14:34,251 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7797): checking classloading for 967acf99df098c28d8f837a9f0746a90 2024-11-24T22:14:34,252 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=9 2024-11-24T22:14:34,252 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure 46b79f5a3db63445091682b5e72f635a, server=bd6592c37bb2,42639,1732486469476 in 578 msec 2024-11-24T22:14:34,256 INFO [StoreOpener-83e58b0e0989e11c83283bf7bd41306c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 83e58b0e0989e11c83283bf7bd41306c columnFamilyName family 2024-11-24T22:14:34,256 DEBUG [StoreOpener-83e58b0e0989e11c83283bf7bd41306c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:34,256 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=46b79f5a3db63445091682b5e72f635a, ASSIGN in 963 msec 2024-11-24T22:14:34,257 INFO [StoreOpener-83e58b0e0989e11c83283bf7bd41306c-1 {}] regionserver.HStore(327): Store=83e58b0e0989e11c83283bf7bd41306c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:14:34,257 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1038): replaying wal for 83e58b0e0989e11c83283bf7bd41306c 2024-11-24T22:14:34,258 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/83e58b0e0989e11c83283bf7bd41306c 2024-11-24T22:14:34,259 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/83e58b0e0989e11c83283bf7bd41306c 2024-11-24T22:14:34,259 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=736f44b8f348e2a5a96dfd3abc5e62b2, regionState=OPEN, openSeqNum=2, regionLocation=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:34,259 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1048): stopping wal replay for 83e58b0e0989e11c83283bf7bd41306c 2024-11-24T22:14:34,259 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1060): Cleaning up temporary data for 83e58b0e0989e11c83283bf7bd41306c 2024-11-24T22:14:34,262 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/dc147c653b22684b8e41092cd6a0c8eb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T22:14:34,262 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1093): writing seq id for 83e58b0e0989e11c83283bf7bd41306c 2024-11-24T22:14:34,263 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened dc147c653b22684b8e41092cd6a0c8eb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73629247, jitterRate=0.09716127812862396}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T22:14:34,263 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for dc147c653b22684b8e41092cd6a0c8eb 2024-11-24T22:14:34,263 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for dc147c653b22684b8e41092cd6a0c8eb: Running coprocessor pre-open hook at 1732486474213Writing region info on filesystem at 1732486474213Initializing all the Stores at 1732486474215 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486474215Cleaning up temporary data from old regions at 1732486474242 (+27 ms)Running coprocessor post-open hooks at 1732486474263 (+21 ms)Region opened successfully at 1732486474263 2024-11-24T22:14:34,263 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=10, state=RUNNABLE, hasLock=true; OpenRegionProcedure 736f44b8f348e2a5a96dfd3abc5e62b2, server=bd6592c37bb2,38125,1732486469405 because future has completed 2024-11-24T22:14:34,264 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1732486472486.dc147c653b22684b8e41092cd6a0c8eb., pid=39, masterSystemTime=1732486473721 2024-11-24T22:14:34,269 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1732486472486.dc147c653b22684b8e41092cd6a0c8eb. 2024-11-24T22:14:34,269 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened test,mmm,1732486472486.dc147c653b22684b8e41092cd6a0c8eb. 2024-11-24T22:14:34,270 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(132): Open test,xxx,1732486472486.516ead6fe1773c4da8708cdc5713f526. 2024-11-24T22:14:34,270 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7752): Opening region: {ENCODED => 516ead6fe1773c4da8708cdc5713f526, NAME => 'test,xxx,1732486472486.516ead6fe1773c4da8708cdc5713f526.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-11-24T22:14:34,270 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 516ead6fe1773c4da8708cdc5713f526 2024-11-24T22:14:34,270 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(898): Instantiated test,xxx,1732486472486.516ead6fe1773c4da8708cdc5713f526.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:34,271 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7794): checking encryption for 516ead6fe1773c4da8708cdc5713f526 2024-11-24T22:14:34,271 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7797): checking classloading for 516ead6fe1773c4da8708cdc5713f526 2024-11-24T22:14:34,276 INFO [StoreOpener-967acf99df098c28d8f837a9f0746a90-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 967acf99df098c28d8f837a9f0746a90 2024-11-24T22:14:34,277 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=10 2024-11-24T22:14:34,277 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 736f44b8f348e2a5a96dfd3abc5e62b2, server=bd6592c37bb2,38125,1732486469405 in 532 msec 2024-11-24T22:14:34,278 INFO [StoreOpener-967acf99df098c28d8f837a9f0746a90-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 967acf99df098c28d8f837a9f0746a90 columnFamilyName family 2024-11-24T22:14:34,278 DEBUG [StoreOpener-967acf99df098c28d8f837a9f0746a90-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:34,279 INFO [StoreOpener-967acf99df098c28d8f837a9f0746a90-1 {}] regionserver.HStore(327): Store=967acf99df098c28d8f837a9f0746a90/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:14:34,279 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1038): replaying wal for 967acf99df098c28d8f837a9f0746a90 2024-11-24T22:14:34,280 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=736f44b8f348e2a5a96dfd3abc5e62b2, ASSIGN in 988 msec 2024-11-24T22:14:34,280 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/967acf99df098c28d8f837a9f0746a90 2024-11-24T22:14:34,281 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/967acf99df098c28d8f837a9f0746a90 2024-11-24T22:14:34,282 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1048): stopping wal replay for 967acf99df098c28d8f837a9f0746a90 2024-11-24T22:14:34,282 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1060): Cleaning up temporary data for 967acf99df098c28d8f837a9f0746a90 2024-11-24T22:14:34,284 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=dc147c653b22684b8e41092cd6a0c8eb, regionState=OPEN, openSeqNum=2, regionLocation=bd6592c37bb2,41619,1732486469269 2024-11-24T22:14:34,284 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1093): writing seq id for 967acf99df098c28d8f837a9f0746a90 2024-11-24T22:14:34,288 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/83e58b0e0989e11c83283bf7bd41306c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T22:14:34,289 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1114): Opened 83e58b0e0989e11c83283bf7bd41306c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65871091, jitterRate=-0.018444254994392395}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T22:14:34,289 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 83e58b0e0989e11c83283bf7bd41306c 2024-11-24T22:14:34,289 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1006): Region open journal for 83e58b0e0989e11c83283bf7bd41306c: Running coprocessor pre-open hook at 1732486474233Writing region info on filesystem at 1732486474233Initializing all the Stores at 1732486474237 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486474237Cleaning up temporary data from old regions at 1732486474259 (+22 ms)Running coprocessor post-open hooks at 1732486474289 (+30 ms)Region opened successfully at 1732486474289 2024-11-24T22:14:34,290 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure dc147c653b22684b8e41092cd6a0c8eb, server=bd6592c37bb2,41619,1732486469269 because future has completed 2024-11-24T22:14:34,291 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2236): Post open deploy tasks for test,qqq,1732486472486.83e58b0e0989e11c83283bf7bd41306c., pid=48, masterSystemTime=1732486473821 2024-11-24T22:14:34,294 INFO [StoreOpener-516ead6fe1773c4da8708cdc5713f526-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 516ead6fe1773c4da8708cdc5713f526 2024-11-24T22:14:34,295 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2266): Finished post open deploy task for test,qqq,1732486472486.83e58b0e0989e11c83283bf7bd41306c. 2024-11-24T22:14:34,295 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(153): Opened test,qqq,1732486472486.83e58b0e0989e11c83283bf7bd41306c. 2024-11-24T22:14:34,298 INFO [StoreOpener-516ead6fe1773c4da8708cdc5713f526-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 516ead6fe1773c4da8708cdc5713f526 columnFamilyName family 2024-11-24T22:14:34,298 DEBUG [StoreOpener-516ead6fe1773c4da8708cdc5713f526-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:34,299 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=17 2024-11-24T22:14:34,299 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure dc147c653b22684b8e41092cd6a0c8eb, server=bd6592c37bb2,41619,1732486469269 in 676 msec 2024-11-24T22:14:34,300 INFO [StoreOpener-516ead6fe1773c4da8708cdc5713f526-1 {}] regionserver.HStore(327): Store=516ead6fe1773c4da8708cdc5713f526/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:14:34,300 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1038): replaying wal for 516ead6fe1773c4da8708cdc5713f526 2024-11-24T22:14:34,301 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/516ead6fe1773c4da8708cdc5713f526 2024-11-24T22:14:34,302 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/516ead6fe1773c4da8708cdc5713f526 2024-11-24T22:14:34,303 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1048): stopping wal replay for 516ead6fe1773c4da8708cdc5713f526 2024-11-24T22:14:34,303 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1060): Cleaning up temporary data for 516ead6fe1773c4da8708cdc5713f526 2024-11-24T22:14:34,305 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN in 1.0100 sec 2024-11-24T22:14:34,306 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/967acf99df098c28d8f837a9f0746a90/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T22:14:34,306 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1114): Opened 967acf99df098c28d8f837a9f0746a90; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67655871, jitterRate=0.008151039481163025}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T22:14:34,307 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 967acf99df098c28d8f837a9f0746a90 2024-11-24T22:14:34,307 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1093): writing seq id for 516ead6fe1773c4da8708cdc5713f526 2024-11-24T22:14:34,307 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1006): Region open journal for 967acf99df098c28d8f837a9f0746a90: Running coprocessor pre-open hook at 1732486474251Writing region info on filesystem at 1732486474251Initializing all the Stores at 1732486474254 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486474254Cleaning up temporary data from old regions at 1732486474282 (+28 ms)Running coprocessor post-open hooks at 1732486474307 (+25 ms)Region opened successfully at 1732486474307 2024-11-24T22:14:34,307 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=83e58b0e0989e11c83283bf7bd41306c, regionState=OPEN, openSeqNum=2, regionLocation=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:34,308 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1732486472486.967acf99df098c28d8f837a9f0746a90., pid=54, masterSystemTime=1732486473897 2024-11-24T22:14:34,313 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1732486472486.967acf99df098c28d8f837a9f0746a90. 2024-11-24T22:14:34,313 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(153): Opened test,lll,1732486472486.967acf99df098c28d8f837a9f0746a90. 2024-11-24T22:14:34,312 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=48, ppid=21, state=RUNNABLE, hasLock=true; OpenRegionProcedure 83e58b0e0989e11c83283bf7bd41306c, server=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:34,313 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(132): Open test,ttt,1732486472486.acdab7e426c93b10b34b4d331b21cef0. 2024-11-24T22:14:34,313 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7752): Opening region: {ENCODED => acdab7e426c93b10b34b4d331b21cef0, NAME => 'test,ttt,1732486472486.acdab7e426c93b10b34b4d331b21cef0.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-24T22:14:34,314 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test acdab7e426c93b10b34b4d331b21cef0 2024-11-24T22:14:34,314 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(898): Instantiated test,ttt,1732486472486.acdab7e426c93b10b34b4d331b21cef0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:34,314 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7794): checking encryption for acdab7e426c93b10b34b4d331b21cef0 2024-11-24T22:14:34,314 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7797): checking classloading for acdab7e426c93b10b34b4d331b21cef0 2024-11-24T22:14:34,321 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=21 2024-11-24T22:14:34,321 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=21, state=SUCCESS, hasLock=false; OpenRegionProcedure 83e58b0e0989e11c83283bf7bd41306c, server=bd6592c37bb2,42639,1732486469476 in 626 msec 2024-11-24T22:14:34,324 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=83e58b0e0989e11c83283bf7bd41306c, ASSIGN in 1.0320 sec 2024-11-24T22:14:34,326 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=967acf99df098c28d8f837a9f0746a90, regionState=OPEN, openSeqNum=2, regionLocation=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:34,329 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 967acf99df098c28d8f837a9f0746a90, server=bd6592c37bb2,38125,1732486469405 because future has completed 2024-11-24T22:14:34,329 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/516ead6fe1773c4da8708cdc5713f526/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T22:14:34,330 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1114): Opened 516ead6fe1773c4da8708cdc5713f526; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71211211, jitterRate=0.06112973392009735}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T22:14:34,330 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 516ead6fe1773c4da8708cdc5713f526 2024-11-24T22:14:34,331 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1006): Region open journal for 516ead6fe1773c4da8708cdc5713f526: Running coprocessor pre-open hook at 1732486474271Writing region info on filesystem at 1732486474271Initializing all the Stores at 1732486474272 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486474273 (+1 ms)Cleaning up temporary data from old regions at 1732486474303 (+30 ms)Running coprocessor post-open hooks at 1732486474331 (+28 ms)Region opened successfully at 1732486474331 2024-11-24T22:14:34,332 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1732486472486.516ead6fe1773c4da8708cdc5713f526., pid=44, masterSystemTime=1732486473721 2024-11-24T22:14:34,335 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1732486472486.516ead6fe1773c4da8708cdc5713f526. 2024-11-24T22:14:34,335 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(153): Opened test,xxx,1732486472486.516ead6fe1773c4da8708cdc5713f526. 2024-11-24T22:14:34,341 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=16 2024-11-24T22:14:34,341 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 967acf99df098c28d8f837a9f0746a90, server=bd6592c37bb2,38125,1732486469405 in 562 msec 2024-11-24T22:14:34,344 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=967acf99df098c28d8f837a9f0746a90, ASSIGN in 1.0520 sec 2024-11-24T22:14:34,344 INFO [StoreOpener-acdab7e426c93b10b34b4d331b21cef0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region acdab7e426c93b10b34b4d331b21cef0 2024-11-24T22:14:34,346 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=516ead6fe1773c4da8708cdc5713f526, regionState=OPEN, openSeqNum=2, regionLocation=bd6592c37bb2,41619,1732486469269 2024-11-24T22:14:34,347 INFO [StoreOpener-acdab7e426c93b10b34b4d331b21cef0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region acdab7e426c93b10b34b4d331b21cef0 columnFamilyName family 2024-11-24T22:14:34,347 DEBUG [StoreOpener-acdab7e426c93b10b34b4d331b21cef0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:34,348 INFO [StoreOpener-acdab7e426c93b10b34b4d331b21cef0-1 {}] regionserver.HStore(327): Store=acdab7e426c93b10b34b4d331b21cef0/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:14:34,348 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1038): replaying wal for acdab7e426c93b10b34b4d331b21cef0 2024-11-24T22:14:34,349 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/acdab7e426c93b10b34b4d331b21cef0 2024-11-24T22:14:34,350 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=44, ppid=28, state=RUNNABLE, hasLock=true; OpenRegionProcedure 516ead6fe1773c4da8708cdc5713f526, server=bd6592c37bb2,41619,1732486469269 2024-11-24T22:14:34,350 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/acdab7e426c93b10b34b4d331b21cef0 2024-11-24T22:14:34,351 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1048): stopping wal replay for acdab7e426c93b10b34b4d331b21cef0 2024-11-24T22:14:34,351 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1060): Cleaning up temporary data for acdab7e426c93b10b34b4d331b21cef0 2024-11-24T22:14:34,355 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1093): writing seq id for acdab7e426c93b10b34b4d331b21cef0 2024-11-24T22:14:34,355 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=28 2024-11-24T22:14:34,356 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=28, state=SUCCESS, hasLock=false; OpenRegionProcedure 516ead6fe1773c4da8708cdc5713f526, server=bd6592c37bb2,41619,1732486469269 in 690 msec 2024-11-24T22:14:34,358 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=516ead6fe1773c4da8708cdc5713f526, ASSIGN in 1.0670 sec 2024-11-24T22:14:34,362 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/acdab7e426c93b10b34b4d331b21cef0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T22:14:34,363 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1114): Opened acdab7e426c93b10b34b4d331b21cef0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62567144, jitterRate=-0.06767690181732178}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T22:14:34,363 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1122): Running coprocessor post-open hooks for acdab7e426c93b10b34b4d331b21cef0 2024-11-24T22:14:34,363 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1006): Region open journal for acdab7e426c93b10b34b4d331b21cef0: Running coprocessor pre-open hook at 1732486474314Writing region info on filesystem at 1732486474314Initializing all the Stores at 1732486474316 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486474316Cleaning up temporary data from old regions at 1732486474351 (+35 ms)Running coprocessor post-open hooks at 1732486474363 (+12 ms)Region opened successfully at 1732486474363 2024-11-24T22:14:34,364 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1732486472486.acdab7e426c93b10b34b4d331b21cef0., pid=56, masterSystemTime=1732486474112 2024-11-24T22:14:34,368 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1732486472486.acdab7e426c93b10b34b4d331b21cef0. 2024-11-24T22:14:34,368 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(153): Opened test,ttt,1732486472486.acdab7e426c93b10b34b4d331b21cef0. 2024-11-24T22:14:34,370 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=acdab7e426c93b10b34b4d331b21cef0, regionState=OPEN, openSeqNum=2, regionLocation=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:34,374 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure acdab7e426c93b10b34b4d331b21cef0, server=bd6592c37bb2,38125,1732486469405 because future has completed 2024-11-24T22:14:34,383 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=24 2024-11-24T22:14:34,383 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure acdab7e426c93b10b34b4d331b21cef0, server=bd6592c37bb2,38125,1732486469405 in 417 msec 2024-11-24T22:14:34,390 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=4 2024-11-24T22:14:34,390 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=acdab7e426c93b10b34b4d331b21cef0, ASSIGN in 1.0940 sec 2024-11-24T22:14:34,391 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T22:14:34,392 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732486474391"}]},"ts":"1732486474391"} 2024-11-24T22:14:34,395 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLED in hbase:meta 2024-11-24T22:14:34,397 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T22:14:34,404 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test in 1.8890 sec 2024-11-24T22:14:34,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36279 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T22:14:34,697 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test completed 2024-11-24T22:14:34,697 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table test get assigned. Timeout = 60000ms 2024-11-24T22:14:34,699 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T22:14:34,722 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table test assigned to meta. Checking AM states. 2024-11-24T22:14:34,723 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T22:14:34,724 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table test assigned. 2024-11-24T22:14:34,725 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T22:14:34,731 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-24T22:14:34,757 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-24T22:14:34,798 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:14:34,839 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60000, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:14:34,843 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:14:34,867 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43712, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:14:34,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36279 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 4279880609a62ccb9e9fb29e79bdf7e7 changed from -1.0 to 0.0, refreshing cache 2024-11-24T22:14:34,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36279 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 736f44b8f348e2a5a96dfd3abc5e62b2 changed from -1.0 to 0.0, refreshing cache 2024-11-24T22:14:34,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36279 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 94f2fa48145fe3bdeffbfded6a4edd73 changed from -1.0 to 0.0, refreshing cache 2024-11-24T22:14:34,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36279 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 967acf99df098c28d8f837a9f0746a90 changed from -1.0 to 0.0, refreshing cache 2024-11-24T22:14:34,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36279 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ce1f0bcb3026cb1cbbd9e63c21b83f99 changed from -1.0 to 0.0, refreshing cache 2024-11-24T22:14:34,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36279 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region acdab7e426c93b10b34b4d331b21cef0 changed from -1.0 to 0.0, refreshing cache 2024-11-24T22:14:34,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36279 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 79c2dd6f654fabefe0fc06b8199e8562 changed from -1.0 to 0.0, refreshing cache 2024-11-24T22:14:34,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36279 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 79d315a497ec3ad891988cb800bae337 changed from -1.0 to 0.0, refreshing cache 2024-11-24T22:14:34,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36279 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 4e91326d9778777879bc895ecb241af8 changed from -1.0 to 0.0, refreshing cache 2024-11-24T22:14:34,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36279 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 698dd9d64bdaf6a81e797d2590ff86f9 changed from -1.0 to 0.0, refreshing cache 2024-11-24T22:14:34,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36279 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f72c935213180e3ff989af795022b7e4 changed from -1.0 to 0.0, refreshing cache 2024-11-24T22:14:34,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36279 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 9b2e9ce57ffd0cd61c72602e5f6d9cae changed from -1.0 to 0.0, refreshing cache 2024-11-24T22:14:34,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36279 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region dc147c653b22684b8e41092cd6a0c8eb changed from -1.0 to 0.0, refreshing cache 2024-11-24T22:14:34,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36279 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region cca6a66154e5c299fad3a247afbfb5fd changed from -1.0 to 0.0, refreshing cache 2024-11-24T22:14:34,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36279 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 84dd509902edc43083bc78fe4b70922c changed from -1.0 to 0.0, refreshing cache 2024-11-24T22:14:34,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36279 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region fc9f96fd4c44240a85e6b2c001e46153 changed from -1.0 to 0.0, refreshing cache 2024-11-24T22:14:34,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36279 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 516ead6fe1773c4da8708cdc5713f526 changed from -1.0 to 0.0, refreshing cache 2024-11-24T22:14:34,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36279 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c68a44a11eeaccf8a3c2f49ba1446c1e changed from -1.0 to 0.0, refreshing cache 2024-11-24T22:14:34,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36279 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 46b79f5a3db63445091682b5e72f635a changed from -1.0 to 0.0, refreshing cache 2024-11-24T22:14:34,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36279 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 72a41015f7b88194cc3a08feb66aa4f9 changed from -1.0 to 0.0, refreshing cache 2024-11-24T22:14:34,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36279 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 0b86957c7b81466a818bbff45fac751f changed from -1.0 to 0.0, refreshing cache 2024-11-24T22:14:34,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36279 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 5160ff1dd0e56b08fbca2ffe581141d4 changed from -1.0 to 0.0, refreshing cache 2024-11-24T22:14:34,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36279 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 83e58b0e0989e11c83283bf7bd41306c changed from -1.0 to 0.0, refreshing cache 2024-11-24T22:14:34,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36279 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c802d82f160c5269119d007388afb85e changed from -1.0 to 0.0, refreshing cache 2024-11-24T22:14:34,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36279 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 50d9b38af2d6413e500860133d150aa8 changed from -1.0 to 0.0, refreshing cache 2024-11-24T22:14:34,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36279 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ed5b70c013e388180062b1513b980a07 changed from -1.0 to 0.0, refreshing cache 2024-11-24T22:14:34,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36279 {}] rsgroup.RSGroupBasedLoadBalancer(138): Balancing RSGroup=default 2024-11-24T22:14:34,980 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36279 {}] rsgroup.RSGroupBasedLoadBalancer(151): Start Generate Balance plan for group: default 2024-11-24T22:14:34,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36279 {}] balancer.BaseLoadBalancer(619): Start Generate Balance plan for cluster. 2024-11-24T22:14:34,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36279 {}] balancer.BalancerClusterState(204): Hosts are {bd6592c37bb2=0} racks are {/default-rack=0} 2024-11-24T22:14:34,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36279 {}] balancer.BalancerClusterState(303): server 0 has 8 regions 2024-11-24T22:14:34,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36279 {}] balancer.BalancerClusterState(303): server 1 has 10 regions 2024-11-24T22:14:34,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36279 {}] balancer.BalancerClusterState(303): server 2 has 9 regions 2024-11-24T22:14:34,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36279 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T22:14:34,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36279 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T22:14:34,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36279 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-24T22:14:34,982 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36279 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T22:14:34,982 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36279 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T22:14:34,982 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36279 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-24T22:14:34,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36279 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1 2024-11-24T22:14:34,988 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36279 {}] balancer.StochasticLoadBalancer(403): Cluster wide - Calculating plan. may take up to 30000ms to complete. 2024-11-24T22:14:34,989 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36279 {}] balancer.StochasticLoadBalancer(515): Start StochasticLoadBalancer.balancer, initial weighted average imbalance=0.06931633435746164, functionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.06415002990995843, need balance); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.0); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); computedMaxSteps=64800 2024-11-24T22:14:35,719 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36279 {}] balancer.StochasticLoadBalancer(562): Finished computing new moving plan. Computation took 735 ms to try 64800 different iterations. Found a solution that moves 1 regions; Going from a computed imbalance of 0.06931633435746164 to a new imbalance of 0.01690158032826896. funtionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.0); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.037037037037037035, need balance); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); 2024-11-24T22:14:35,723 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36279 {}] master.HMaster(2167): Balancer plans size is 1, the balance interval is 300000 ms, and the max number regions in transition is 27 2024-11-24T22:14:35,723 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36279 {}] master.HMaster(2172): balance hri=cca6a66154e5c299fad3a247afbfb5fd, source=bd6592c37bb2,41619,1732486469269, destination=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:35,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36279 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test, region=cca6a66154e5c299fad3a247afbfb5fd, REOPEN/MOVE 2024-11-24T22:14:35,726 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test, region=cca6a66154e5c299fad3a247afbfb5fd, REOPEN/MOVE 2024-11-24T22:14:35,728 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=cca6a66154e5c299fad3a247afbfb5fd, regionState=CLOSING, regionLocation=bd6592c37bb2,41619,1732486469269 2024-11-24T22:14:35,733 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=test, region=cca6a66154e5c299fad3a247afbfb5fd, REOPEN/MOVE because future has completed 2024-11-24T22:14:35,734 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T22:14:35,734 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE, hasLock=false; CloseRegionProcedure cca6a66154e5c299fad3a247afbfb5fd, server=bd6592c37bb2,41619,1732486469269}] 2024-11-24T22:14:35,895 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(122): Close cca6a66154e5c299fad3a247afbfb5fd 2024-11-24T22:14:35,895 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T22:14:35,896 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1722): Closing cca6a66154e5c299fad3a247afbfb5fd, disabling compactions & flushes 2024-11-24T22:14:35,896 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1755): Closing region test,ppp,1732486472486.cca6a66154e5c299fad3a247afbfb5fd. 2024-11-24T22:14:35,896 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1732486472486.cca6a66154e5c299fad3a247afbfb5fd. 2024-11-24T22:14:35,896 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1732486472486.cca6a66154e5c299fad3a247afbfb5fd. after waiting 0 ms 2024-11-24T22:14:35,896 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1732486472486.cca6a66154e5c299fad3a247afbfb5fd. 2024-11-24T22:14:35,911 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/cca6a66154e5c299fad3a247afbfb5fd/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T22:14:35,915 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1973): Closed test,ppp,1732486472486.cca6a66154e5c299fad3a247afbfb5fd. 2024-11-24T22:14:35,915 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1676): Region close journal for cca6a66154e5c299fad3a247afbfb5fd: Waiting for close lock at 1732486475896Running coprocessor pre-close hooks at 1732486475896Disabling compacts and flushes for region at 1732486475896Disabling writes for close at 1732486475896Writing region close event to WAL at 1732486475900 (+4 ms)Running coprocessor post-close hooks at 1732486475912 (+12 ms)Closed at 1732486475915 (+3 ms) 2024-11-24T22:14:35,916 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegionServer(3302): Adding cca6a66154e5c299fad3a247afbfb5fd move to bd6592c37bb2,38125,1732486469405 record at close sequenceid=2 2024-11-24T22:14:35,922 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(157): Closed cca6a66154e5c299fad3a247afbfb5fd 2024-11-24T22:14:35,923 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=cca6a66154e5c299fad3a247afbfb5fd, regionState=CLOSED 2024-11-24T22:14:35,926 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE, hasLock=false; CloseRegionProcedure cca6a66154e5c299fad3a247afbfb5fd, server=bd6592c37bb2,41619,1732486469269 because future has completed 2024-11-24T22:14:35,932 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-24T22:14:35,932 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; CloseRegionProcedure cca6a66154e5c299fad3a247afbfb5fd, server=bd6592c37bb2,41619,1732486469269 in 194 msec 2024-11-24T22:14:35,935 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=cca6a66154e5c299fad3a247afbfb5fd, REOPEN/MOVE; state=CLOSED, location=bd6592c37bb2,38125,1732486469405; forceNewPlan=false, retain=false 2024-11-24T22:14:36,086 INFO [bd6592c37bb2:36279 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-24T22:14:36,086 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=cca6a66154e5c299fad3a247afbfb5fd, regionState=OPENING, regionLocation=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:36,089 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=cca6a66154e5c299fad3a247afbfb5fd, REOPEN/MOVE because future has completed 2024-11-24T22:14:36,090 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=57, state=RUNNABLE, hasLock=false; OpenRegionProcedure cca6a66154e5c299fad3a247afbfb5fd, server=bd6592c37bb2,38125,1732486469405}] 2024-11-24T22:14:36,251 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(132): Open test,ppp,1732486472486.cca6a66154e5c299fad3a247afbfb5fd. 2024-11-24T22:14:36,252 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7752): Opening region: {ENCODED => cca6a66154e5c299fad3a247afbfb5fd, NAME => 'test,ppp,1732486472486.cca6a66154e5c299fad3a247afbfb5fd.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-24T22:14:36,252 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test cca6a66154e5c299fad3a247afbfb5fd 2024-11-24T22:14:36,252 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(898): Instantiated test,ppp,1732486472486.cca6a66154e5c299fad3a247afbfb5fd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:36,252 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7794): checking encryption for cca6a66154e5c299fad3a247afbfb5fd 2024-11-24T22:14:36,253 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7797): checking classloading for cca6a66154e5c299fad3a247afbfb5fd 2024-11-24T22:14:36,255 INFO [StoreOpener-cca6a66154e5c299fad3a247afbfb5fd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region cca6a66154e5c299fad3a247afbfb5fd 2024-11-24T22:14:36,257 INFO [StoreOpener-cca6a66154e5c299fad3a247afbfb5fd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cca6a66154e5c299fad3a247afbfb5fd columnFamilyName family 2024-11-24T22:14:36,257 DEBUG [StoreOpener-cca6a66154e5c299fad3a247afbfb5fd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:36,258 INFO [StoreOpener-cca6a66154e5c299fad3a247afbfb5fd-1 {}] regionserver.HStore(327): Store=cca6a66154e5c299fad3a247afbfb5fd/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:14:36,258 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1038): replaying wal for cca6a66154e5c299fad3a247afbfb5fd 2024-11-24T22:14:36,259 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/cca6a66154e5c299fad3a247afbfb5fd 2024-11-24T22:14:36,262 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/cca6a66154e5c299fad3a247afbfb5fd 2024-11-24T22:14:36,263 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1048): stopping wal replay for cca6a66154e5c299fad3a247afbfb5fd 2024-11-24T22:14:36,263 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1060): Cleaning up temporary data for cca6a66154e5c299fad3a247afbfb5fd 2024-11-24T22:14:36,266 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1093): writing seq id for cca6a66154e5c299fad3a247afbfb5fd 2024-11-24T22:14:36,267 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1114): Opened cca6a66154e5c299fad3a247afbfb5fd; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64716507, jitterRate=-0.0356488972902298}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T22:14:36,267 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cca6a66154e5c299fad3a247afbfb5fd 2024-11-24T22:14:36,267 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1006): Region open journal for cca6a66154e5c299fad3a247afbfb5fd: Running coprocessor pre-open hook at 1732486476253Writing region info on filesystem at 1732486476253Initializing all the Stores at 1732486476254 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486476254Cleaning up temporary data from old regions at 1732486476263 (+9 ms)Running coprocessor post-open hooks at 1732486476267 (+4 ms)Region opened successfully at 1732486476267 2024-11-24T22:14:36,269 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1732486472486.cca6a66154e5c299fad3a247afbfb5fd., pid=59, masterSystemTime=1732486476245 2024-11-24T22:14:36,274 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ppp,1732486472486.cca6a66154e5c299fad3a247afbfb5fd. 2024-11-24T22:14:36,274 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(153): Opened test,ppp,1732486472486.cca6a66154e5c299fad3a247afbfb5fd. 2024-11-24T22:14:36,276 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=cca6a66154e5c299fad3a247afbfb5fd, regionState=OPEN, openSeqNum=5, regionLocation=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:36,280 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=57, state=RUNNABLE, hasLock=false; OpenRegionProcedure cca6a66154e5c299fad3a247afbfb5fd, server=bd6592c37bb2,38125,1732486469405 because future has completed 2024-11-24T22:14:36,285 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=57 2024-11-24T22:14:36,285 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=57, state=SUCCESS, hasLock=false; OpenRegionProcedure cca6a66154e5c299fad3a247afbfb5fd, server=bd6592c37bb2,38125,1732486469405 in 192 msec 2024-11-24T22:14:36,288 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=cca6a66154e5c299fad3a247afbfb5fd, REOPEN/MOVE in 561 msec 2024-11-24T22:14:36,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36279 {}] master.HMaster(2203): Balancer is going into sleep until next period in 300000ms 2024-11-24T22:14:36,330 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [900,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T22:14:36,343 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36279 {}] master.MasterRpcServices(567): Client=jenkins//172.17.0.2 set balanceSwitch=false 2024-11-24T22:14:36,371 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.assignment.TestRollbackSCP#testFailAndRollback Thread=381, OpenFileDescriptor=646, MaxFileDescriptor=1048576, SystemLoadAverage=445, ProcessCount=11, AvailableMemoryMB=6451 2024-11-24T22:14:36,376 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table hbase:meta 2024-11-24T22:14:36,376 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=hbase:meta,,1 2024-11-24T22:14:36,377 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing bd6592c37bb2,41619,1732486469269 2024-11-24T22:14:36,378 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server bd6592c37bb2,41619,1732486469269: Simulated kill ***** 2024-11-24T22:14:36,378 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-24T22:14:36,381 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-24T22:14:36,384 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-24T22:14:36,385 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-24T22:14:36,386 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-24T22:14:36,386 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 334491904 }, "NonHeapMemoryUsage": { "committed": 153288704, "init": 7667712, "max": -1, "used": 150924600 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "bd6592c37bb2" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "bd6592c37bb2" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.Context": "regionserver", "tag.Hostname": "bd6592c37bb2" } ] } 2024-11-24T22:14:36,389 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36279 {}] master.MasterRpcServices(700): bd6592c37bb2,41619,1732486469269 reported a fatal error: ***** ABORTING region server bd6592c37bb2,41619,1732486469269: Simulated kill ***** 2024-11-24T22:14:36,391 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'bd6592c37bb2,41619,1732486469269' ***** 2024-11-24T22:14:36,392 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-11-24T22:14:36,392 INFO [RS:0;bd6592c37bb2:41619 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T22:14:36,392 INFO [RS:0;bd6592c37bb2:41619 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-24T22:14:36,392 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T22:14:36,392 INFO [RS:0;bd6592c37bb2:41619 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-24T22:14:36,393 INFO [RS:0;bd6592c37bb2:41619 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T22:14:36,393 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [15,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T22:14:36,393 INFO [RS:0;bd6592c37bb2:41619 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;bd6592c37bb2:41619. 2024-11-24T22:14:36,393 DEBUG [RS:0;bd6592c37bb2:41619 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T22:14:36,401 DEBUG [RS:0;bd6592c37bb2:41619 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:36,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35383 is added to blk_1073741836_1012 (size=33749) 2024-11-24T22:14:36,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34013 is added to blk_1073741836_1012 (size=33749) 2024-11-24T22:14:36,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741836_1012 (size=33749) 2024-11-24T22:14:36,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35383 is added to blk_1073741834_1010 (size=2769) 2024-11-24T22:14:36,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741834_1010 (size=2769) 2024-11-24T22:14:36,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34013 is added to blk_1073741834_1010 (size=2769) 2024-11-24T22:14:36,436 DEBUG [RS:0;bd6592c37bb2:41619 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:36,436 INFO [RS:0;bd6592c37bb2:41619 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T22:14:36,436 INFO [RS:0;bd6592c37bb2:41619 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41619 2024-11-24T22:14:36,447 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36279-0x1014c0b79ef0000, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T22:14:36,447 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41619-0x1014c0b79ef0001, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/bd6592c37bb2,41619,1732486469269 2024-11-24T22:14:36,448 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [bd6592c37bb2,41619,1732486469269] 2024-11-24T22:14:36,450 INFO [RS:0;bd6592c37bb2:41619 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T22:14:36,451 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/bd6592c37bb2,41619,1732486469269 already deleted, retry=false 2024-11-24T22:14:36,451 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of bd6592c37bb2,41619,1732486469269 on bd6592c37bb2,36279,1732486468255 2024-11-24T22:14:36,459 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true 2024-11-24T22:14:36,465 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=60, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true 2024-11-24T22:14:36,467 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=60, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true, isMeta: true 2024-11-24T22:14:36,467 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=60 for bd6592c37bb2,41619,1732486469269 (carryingMeta=true) bd6592c37bb2,41619,1732486469269/CRASHED/regionCount=9/lock=java.util.concurrent.locks.ReentrantReadWriteLock@690c5222[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-24T22:14:36,470 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/WALs/bd6592c37bb2,41619,1732486469269-splitting 2024-11-24T22:14:36,472 INFO [PEWorker-1 {}] master.SplitWALManager(105): bd6592c37bb2,41619,1732486469269 WAL count=1, meta=true 2024-11-24T22:14:36,475 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta}] 2024-11-24T22:14:36,483 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:36,484 INFO [regionserver/bd6592c37bb2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T22:14:36,486 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405}] 2024-11-24T22:14:36,494 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T22:14:36,561 INFO [RS:0;bd6592c37bb2:41619 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T22:14:36,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41619-0x1014c0b79ef0001, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T22:14:36,562 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41619-0x1014c0b79ef0001, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T22:14:36,562 INFO [RS:0;bd6592c37bb2:41619 {}] regionserver.HRegionServer(1031): Exiting; stopping=bd6592c37bb2,41619,1732486469269; zookeeper connection closed. 2024-11-24T22:14:36,562 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@f653ece {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@f653ece 2024-11-24T22:14:36,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38125 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=62 2024-11-24T22:14:36,687 INFO [RS_LOG_REPLAY_OPS-regionserver/bd6592c37bb2:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(299): Splitting hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/WALs/bd6592c37bb2,41619,1732486469269-splitting/bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, size=33.0 K (33749bytes) 2024-11-24T22:14:36,688 DEBUG [RS_LOG_REPLAY_OPS-regionserver/bd6592c37bb2:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-24T22:14:36,688 DEBUG [RS_LOG_REPLAY_OPS-regionserver/bd6592c37bb2:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-24T22:14:36,689 INFO [RS_LOG_REPLAY_OPS-regionserver/bd6592c37bb2:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/WALs/bd6592c37bb2,41619,1732486469269-splitting/bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta 2024-11-24T22:14:36,696 INFO [RS_LOG_REPLAY_OPS-regionserver/bd6592c37bb2:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/WALs/bd6592c37bb2,41619,1732486469269-splitting/bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta after 4ms 2024-11-24T22:14:36,714 INFO [RS_LOG_REPLAY_OPS-regionserver/bd6592c37bb2:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(310): Open hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/WALs/bd6592c37bb2,41619,1732486469269-splitting/bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta took 28ms 2024-11-24T22:14:36,728 DEBUG [RS_LOG_REPLAY_OPS-regionserver/bd6592c37bb2:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-24T22:14:36,740 INFO [RS_LOG_REPLAY_OPS-regionserver/bd6592c37bb2:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta.temp 2024-11-24T22:14:36,745 DEBUG [RS_LOG_REPLAY_OPS-regionserver/bd6592c37bb2:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/WALs/bd6592c37bb2,41619,1732486469269-splitting/bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta so closing down 2024-11-24T22:14:36,745 DEBUG [RS_LOG_REPLAY_OPS-regionserver/bd6592c37bb2:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T22:14:36,750 INFO [RS_LOG_REPLAY_OPS-regionserver/bd6592c37bb2:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta.temp 2024-11-24T22:14:36,755 INFO [RS_LOG_REPLAY_OPS-regionserver/bd6592c37bb2:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T22:14:36,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35383 is added to blk_1073741864_1040 (size=33508) 2024-11-24T22:14:36,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34013 is added to blk_1073741864_1040 (size=33508) 2024-11-24T22:14:36,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741864_1040 (size=33508) 2024-11-24T22:14:36,785 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta.temp (wrote 61 edits, skipped 0 edits in 4 ms) 2024-11-24T22:14:36,790 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta.temp to hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-24T22:14:36,798 INFO [RS_LOG_REPLAY_OPS-regionserver/bd6592c37bb2:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(425): Processed 62 edits across 1 Regions in 73 ms; skipped=1; WAL=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/WALs/bd6592c37bb2,41619,1732486469269-splitting/bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, size=33.0 K, length=33749, corrupted=false, cancelled=false 2024-11-24T22:14:36,798 DEBUG [RS_LOG_REPLAY_OPS-regionserver/bd6592c37bb2:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(428): Completed split of hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/WALs/bd6592c37bb2,41619,1732486469269-splitting/bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, journal: Splitting hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/WALs/bd6592c37bb2,41619,1732486469269-splitting/bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, size=33.0 K (33749bytes) at 1732486476687Finishing writing output for hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/WALs/bd6592c37bb2,41619,1732486469269-splitting/bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta so closing down at 1732486476745 (+58 ms)Creating recovered edits writer path=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta.temp at 1732486476750 (+5 ms)3 split writer threads finished at 1732486476755 (+5 ms)Closed recovered edits writer path=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta.temp (wrote 61 edits, skipped 0 edits in 4 ms) at 1732486476785 (+30 ms)Rename recovered edits hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta.temp to hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 at 1732486476790 (+5 ms)Processed 62 edits across 1 Regions in 73 ms; skipped=1; WAL=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/WALs/bd6592c37bb2,41619,1732486469269-splitting/bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, size=33.0 K, length=33749, corrupted=false, cancelled=false at 1732486476798 (+8 ms) 2024-11-24T22:14:36,799 DEBUG [RS_LOG_REPLAY_OPS-regionserver/bd6592c37bb2:0-0 {event_type=RS_LOG_REPLAY, pid=62}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/WALs/bd6592c37bb2,41619,1732486469269-splitting/bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta 2024-11-24T22:14:36,801 DEBUG [RS_LOG_REPLAY_OPS-regionserver/bd6592c37bb2:0-0 {event_type=RS_LOG_REPLAY, pid=62}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=62 2024-11-24T22:14:36,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36279 {}] master.HMaster(4169): Remote procedure done, pid=62 2024-11-24T22:14:36,816 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/WALs/bd6592c37bb2,41619,1732486469269-splitting/bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta to hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/oldWALs 2024-11-24T22:14:36,819 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-24T22:14:36,819 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 in 331 msec 2024-11-24T22:14:36,820 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:36,827 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-24T22:14:36,827 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 in 347 msec 2024-11-24T22:14:36,830 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/WALs/bd6592c37bb2,41619,1732486469269-splitting dir is empty, no logs to split. 2024-11-24T22:14:36,830 INFO [PEWorker-1 {}] master.SplitWALManager(105): bd6592c37bb2,41619,1732486469269 WAL count=0, meta=true 2024-11-24T22:14:36,830 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if bd6592c37bb2,41619,1732486469269 WAL splitting is done? wals=0, meta=true 2024-11-24T22:14:36,832 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-24T22:14:36,833 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T22:14:36,835 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-24T22:14:36,986 DEBUG [bd6592c37bb2:36279 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-24T22:14:36,986 DEBUG [bd6592c37bb2:36279 {}] balancer.BalancerClusterState(204): Hosts are {bd6592c37bb2=0} racks are {/default-rack=0} 2024-11-24T22:14:36,986 DEBUG [bd6592c37bb2:36279 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T22:14:36,986 DEBUG [bd6592c37bb2:36279 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T22:14:36,986 DEBUG [bd6592c37bb2:36279 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T22:14:36,986 DEBUG [bd6592c37bb2:36279 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T22:14:36,986 INFO [bd6592c37bb2:36279 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T22:14:36,986 INFO [bd6592c37bb2:36279 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T22:14:36,986 DEBUG [bd6592c37bb2:36279 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T22:14:36,987 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=63 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:36,988 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bd6592c37bb2,42639,1732486469476, state=OPENING 2024-11-24T22:14:36,991 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38125-0x1014c0b79ef0002, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:36,991 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42639-0x1014c0b79ef0003, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:36,991 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36279-0x1014c0b79ef0000, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:36,991 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:36,991 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:36,991 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T22:14:36,991 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:36,991 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=bd6592c37bb2,42639,1732486469476}] 2024-11-24T22:14:37,161 INFO [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=64}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-24T22:14:37,161 INFO [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T22:14:37,161 INFO [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=64}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-24T22:14:37,168 INFO [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=bd6592c37bb2%2C42639%2C1732486469476.meta, suffix=.meta, logDir=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/WALs/bd6592c37bb2,42639,1732486469476, archiveDir=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/oldWALs, maxLogs=32 2024-11-24T22:14:37,192 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=64}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/WALs/bd6592c37bb2,42639,1732486469476/bd6592c37bb2%2C42639%2C1732486469476.meta.1732486477169.meta, exclude list is [], retry=0 2024-11-24T22:14:37,197 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34013,DS-e98687a1-55aa-42d3-a359-80f43904e747,DISK] 2024-11-24T22:14:37,197 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35383,DS-56b705de-da8a-4837-9f73-0e8d9391298c,DISK] 2024-11-24T22:14:37,197 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34899,DS-fb965c9f-83ab-43e9-b4f1-51c95ecaf7db,DISK] 2024-11-24T22:14:37,212 INFO [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/WALs/bd6592c37bb2,42639,1732486469476/bd6592c37bb2%2C42639%2C1732486469476.meta.1732486477169.meta 2024-11-24T22:14:37,216 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36251:36251),(127.0.0.1/127.0.0.1:38755:38755),(127.0.0.1/127.0.0.1:42065:42065)] 2024-11-24T22:14:37,217 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-24T22:14:37,217 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=64}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-24T22:14:37,217 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-24T22:14:37,218 INFO [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-24T22:14:37,218 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-24T22:14:37,218 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:37,218 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-24T22:14:37,218 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-24T22:14:37,222 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-24T22:14:37,224 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T22:14:37,224 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:37,225 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T22:14:37,225 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-24T22:14:37,227 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T22:14:37,227 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:37,228 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T22:14:37,228 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-24T22:14:37,229 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T22:14:37,229 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:37,230 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T22:14:37,230 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-24T22:14:37,231 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T22:14:37,231 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:37,233 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T22:14:37,233 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T22:14:37,240 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/hbase/meta/1588230740 2024-11-24T22:14:37,244 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/hbase/meta/1588230740 2024-11-24T22:14:37,245 INFO [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-24T22:14:37,283 INFO [regionserver/bd6592c37bb2:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-24T22:14:37,283 INFO [regionserver/bd6592c37bb2:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-24T22:14:37,286 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5793): Applied 277, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=64, path=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-24T22:14:37,315 INFO [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=29.41 KB heapSize=50.67 KB 2024-11-24T22:14:37,460 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/hbase/meta/1588230740/.tmp/info/26a18afce1174d38a5d70c888ce2deb4 is 138, key is test,bbb,1732486472486.4279880609a62ccb9e9fb29e79bdf7e7./info:regioninfo/1732486474002/Put/seqid=0 2024-11-24T22:14:37,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35383 is added to blk_1073741866_1042 (size=36371) 2024-11-24T22:14:37,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741866_1042 (size=36371) 2024-11-24T22:14:37,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34013 is added to blk_1073741866_1042 (size=36371) 2024-11-24T22:14:37,494 INFO [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.26 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/hbase/meta/1588230740/.tmp/info/26a18afce1174d38a5d70c888ce2deb4 2024-11-24T22:14:37,513 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T22:14:37,601 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test' 2024-11-24T22:14:37,611 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-24T22:14:37,629 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/hbase/meta/1588230740/.tmp/ns/c6dc0128d57b4a13abb9bfdc135259f8 is 43, key is default/ns:d/1732486472172/Put/seqid=0 2024-11-24T22:14:37,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34013 is added to blk_1073741867_1043 (size=5153) 2024-11-24T22:14:37,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35383 is added to blk_1073741867_1043 (size=5153) 2024-11-24T22:14:37,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741867_1043 (size=5153) 2024-11-24T22:14:37,683 INFO [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/hbase/meta/1588230740/.tmp/ns/c6dc0128d57b4a13abb9bfdc135259f8 2024-11-24T22:14:37,741 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/hbase/meta/1588230740/.tmp/table/f1d32550b951400cb9ed623fa5a33bfc is 40, key is test/table:state/1732486474391/Put/seqid=0 2024-11-24T22:14:37,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35383 is added to blk_1073741868_1044 (size=5165) 2024-11-24T22:14:37,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741868_1044 (size=5165) 2024-11-24T22:14:37,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34013 is added to blk_1073741868_1044 (size=5165) 2024-11-24T22:14:37,793 INFO [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=72 B at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/hbase/meta/1588230740/.tmp/table/f1d32550b951400cb9ed623fa5a33bfc 2024-11-24T22:14:37,813 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/hbase/meta/1588230740/.tmp/info/26a18afce1174d38a5d70c888ce2deb4 as hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/hbase/meta/1588230740/info/26a18afce1174d38a5d70c888ce2deb4 2024-11-24T22:14:37,825 INFO [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/hbase/meta/1588230740/info/26a18afce1174d38a5d70c888ce2deb4, entries=273, sequenceid=64, filesize=35.5 K 2024-11-24T22:14:37,839 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/hbase/meta/1588230740/.tmp/ns/c6dc0128d57b4a13abb9bfdc135259f8 as hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/hbase/meta/1588230740/ns/c6dc0128d57b4a13abb9bfdc135259f8 2024-11-24T22:14:37,853 INFO [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/hbase/meta/1588230740/ns/c6dc0128d57b4a13abb9bfdc135259f8, entries=2, sequenceid=64, filesize=5.0 K 2024-11-24T22:14:37,864 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/hbase/meta/1588230740/.tmp/table/f1d32550b951400cb9ed623fa5a33bfc as hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/hbase/meta/1588230740/table/f1d32550b951400cb9ed623fa5a33bfc 2024-11-24T22:14:37,880 INFO [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/hbase/meta/1588230740/table/f1d32550b951400cb9ed623fa5a33bfc, entries=2, sequenceid=64, filesize=5.0 K 2024-11-24T22:14:37,887 INFO [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(3140): Finished flush of dataSize ~29.41 KB/30111, heapSize ~50.38 KB/51584, currentSize=0 B/0 for 1588230740 in 594ms, sequenceid=64, compaction requested=false; wal=null 2024-11-24T22:14:37,889 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-24T22:14:37,891 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T22:14:37,891 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T22:14:37,893 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T22:14:37,895 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T22:14:37,911 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/hbase/meta/1588230740/recovered.edits/64.seqid, newMaxSeqId=64, maxSeqId=1 2024-11-24T22:14:37,913 INFO [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=65; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70169527, jitterRate=0.04560743272304535}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T22:14:37,913 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-24T22:14:37,914 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732486477219Writing region info on filesystem at 1732486477219Initializing all the Stores at 1732486477221 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732486477221Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732486477222 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486477222Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732486477222Obtaining lock to block concurrent updates at 1732486477315 (+93 ms)Preparing flush snapshotting stores in 1588230740 at 1732486477315Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=30111, getHeapSize=51824, getOffHeapSize=0, getCellsCount=277 at 1732486477319 (+4 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732486477319Flushing 1588230740/info: creating writer at 1732486477321 (+2 ms)Flushing 1588230740/info: appending metadata at 1732486477437 (+116 ms)Flushing 1588230740/info: closing flushed file at 1732486477443 (+6 ms)Flushing 1588230740/ns: creating writer at 1732486477599 (+156 ms)Flushing 1588230740/ns: appending metadata at 1732486477625 (+26 ms)Flushing 1588230740/ns: closing flushed file at 1732486477626 (+1 ms)Flushing 1588230740/table: creating writer at 1732486477699 (+73 ms)Flushing 1588230740/table: appending metadata at 1732486477740 (+41 ms)Flushing 1588230740/table: closing flushed file at 1732486477740Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@15a19f44: reopening flushed file at 1732486477811 (+71 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@119f8fec: reopening flushed file at 1732486477830 (+19 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5b3cb2c: reopening flushed file at 1732486477854 (+24 ms)Finished flush of dataSize ~29.41 KB/30111, heapSize ~50.38 KB/51584, currentSize=0 B/0 for 1588230740 in 594ms, sequenceid=64, compaction requested=false; wal=null at 1732486477887 (+33 ms)Cleaning up temporary data from old regions at 1732486477891 (+4 ms)Running coprocessor post-open hooks at 1732486477913 (+22 ms)Region opened successfully at 1732486477914 (+1 ms) 2024-11-24T22:14:37,916 INFO [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=64, masterSystemTime=1732486477145 2024-11-24T22:14:37,919 DEBUG [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-24T22:14:37,919 INFO [RS_OPEN_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_META, pid=64}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-24T22:14:37,921 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=63 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=65, regionLocation=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:37,923 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bd6592c37bb2,42639,1732486469476, state=OPEN 2024-11-24T22:14:37,925 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42639-0x1014c0b79ef0003, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:37,925 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38125-0x1014c0b79ef0002, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:37,925 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36279-0x1014c0b79ef0000, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:37,926 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=64, ppid=63, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:37,926 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:37,926 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:37,926 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:37,934 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-24T22:14:37,934 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bd6592c37bb2,42639,1732486469476 in 935 msec 2024-11-24T22:14:37,941 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=60 2024-11-24T22:14:37,941 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1.1020 sec 2024-11-24T22:14:37,942 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): bd6592c37bb2,41619,1732486469269 had 9 regions 2024-11-24T22:14:37,943 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=60, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true, isMeta: false 2024-11-24T22:14:37,950 INFO [PEWorker-1 {}] master.SplitWALManager(105): bd6592c37bb2,41619,1732486469269 WAL count=1, meta=false 2024-11-24T22:14:37,951 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=60, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450}] 2024-11-24T22:14:37,954 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:37,962 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476}] 2024-11-24T22:14:38,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42639 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=66 2024-11-24T22:14:38,154 INFO [RS_LOG_REPLAY_OPS-regionserver/bd6592c37bb2:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(299): Splitting hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/WALs/bd6592c37bb2,41619,1732486469269-splitting/bd6592c37bb2%2C41619%2C1732486469269.1732486471450, size=2.7 K (2769bytes) 2024-11-24T22:14:38,154 INFO [RS_LOG_REPLAY_OPS-regionserver/bd6592c37bb2:0-0 {event_type=RS_LOG_REPLAY, pid=66}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/WALs/bd6592c37bb2,41619,1732486469269-splitting/bd6592c37bb2%2C41619%2C1732486469269.1732486471450 2024-11-24T22:14:38,155 INFO [RS_LOG_REPLAY_OPS-regionserver/bd6592c37bb2:0-0 {event_type=RS_LOG_REPLAY, pid=66}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/WALs/bd6592c37bb2,41619,1732486469269-splitting/bd6592c37bb2%2C41619%2C1732486469269.1732486471450 after 1ms 2024-11-24T22:14:38,165 INFO [RS_LOG_REPLAY_OPS-regionserver/bd6592c37bb2:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(310): Open hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/WALs/bd6592c37bb2,41619,1732486469269-splitting/bd6592c37bb2%2C41619%2C1732486469269.1732486471450 took 11ms 2024-11-24T22:14:38,172 DEBUG [RS_LOG_REPLAY_OPS-regionserver/bd6592c37bb2:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for cca6a66154e5c299fad3a247afbfb5fd: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-24T22:14:38,175 DEBUG [RS_LOG_REPLAY_OPS-regionserver/bd6592c37bb2:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 698dd9d64bdaf6a81e797d2590ff86f9: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-24T22:14:38,187 DEBUG [RS_LOG_REPLAY_OPS-regionserver/bd6592c37bb2:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 84dd509902edc43083bc78fe4b70922c: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-24T22:14:38,189 DEBUG [RS_LOG_REPLAY_OPS-regionserver/bd6592c37bb2:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for f72c935213180e3ff989af795022b7e4: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-24T22:14:38,196 DEBUG [RS_LOG_REPLAY_OPS-regionserver/bd6592c37bb2:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for fc9f96fd4c44240a85e6b2c001e46153: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-24T22:14:38,198 DEBUG [RS_LOG_REPLAY_OPS-regionserver/bd6592c37bb2:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 9b2e9ce57ffd0cd61c72602e5f6d9cae: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-24T22:14:38,201 DEBUG [RS_LOG_REPLAY_OPS-regionserver/bd6592c37bb2:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 4e91326d9778777879bc895ecb241af8: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-24T22:14:38,203 DEBUG [RS_LOG_REPLAY_OPS-regionserver/bd6592c37bb2:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for dc147c653b22684b8e41092cd6a0c8eb: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-24T22:14:38,204 DEBUG [RS_LOG_REPLAY_OPS-regionserver/bd6592c37bb2:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 516ead6fe1773c4da8708cdc5713f526: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-24T22:14:38,205 DEBUG [RS_LOG_REPLAY_OPS-regionserver/bd6592c37bb2:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/WALs/bd6592c37bb2,41619,1732486469269-splitting/bd6592c37bb2%2C41619%2C1732486469269.1732486471450 so closing down 2024-11-24T22:14:38,205 DEBUG [RS_LOG_REPLAY_OPS-regionserver/bd6592c37bb2:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T22:14:38,205 INFO [RS_LOG_REPLAY_OPS-regionserver/bd6592c37bb2:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T22:14:38,205 INFO [RS_LOG_REPLAY_OPS-regionserver/bd6592c37bb2:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(425): Processed 10 edits across 0 Regions in 37 ms; skipped=10; WAL=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/WALs/bd6592c37bb2,41619,1732486469269-splitting/bd6592c37bb2%2C41619%2C1732486469269.1732486471450, size=2.7 K, length=2769, corrupted=false, cancelled=false 2024-11-24T22:14:38,205 DEBUG [RS_LOG_REPLAY_OPS-regionserver/bd6592c37bb2:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(428): Completed split of hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/WALs/bd6592c37bb2,41619,1732486469269-splitting/bd6592c37bb2%2C41619%2C1732486469269.1732486471450, journal: Splitting hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/WALs/bd6592c37bb2,41619,1732486469269-splitting/bd6592c37bb2%2C41619%2C1732486469269.1732486471450, size=2.7 K (2769bytes) at 1732486478154Finishing writing output for hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/WALs/bd6592c37bb2,41619,1732486469269-splitting/bd6592c37bb2%2C41619%2C1732486469269.1732486471450 so closing down at 1732486478205 (+51 ms)3 split writer threads finished at 1732486478205Processed 10 edits across 0 Regions in 37 ms; skipped=10; WAL=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/WALs/bd6592c37bb2,41619,1732486469269-splitting/bd6592c37bb2%2C41619%2C1732486469269.1732486471450, size=2.7 K, length=2769, corrupted=false, cancelled=false at 1732486478205 2024-11-24T22:14:38,205 DEBUG [RS_LOG_REPLAY_OPS-regionserver/bd6592c37bb2:0-0 {event_type=RS_LOG_REPLAY, pid=66}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/WALs/bd6592c37bb2,41619,1732486469269-splitting/bd6592c37bb2%2C41619%2C1732486469269.1732486471450 2024-11-24T22:14:38,205 DEBUG [RS_LOG_REPLAY_OPS-regionserver/bd6592c37bb2:0-0 {event_type=RS_LOG_REPLAY, pid=66}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=66 2024-11-24T22:14:38,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36279 {}] master.HMaster(4169): Remote procedure done, pid=66 2024-11-24T22:14:38,213 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/WALs/bd6592c37bb2,41619,1732486469269-splitting/bd6592c37bb2%2C41619%2C1732486469269.1732486471450 to hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/oldWALs 2024-11-24T22:14:38,217 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=66, resume processing ppid=65 2024-11-24T22:14:38,218 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 in 252 msec 2024-11-24T22:14:38,219 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:38,228 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=60 2024-11-24T22:14:38,229 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 in 268 msec 2024-11-24T22:14:38,231 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/WALs/bd6592c37bb2,41619,1732486469269-splitting dir is empty, no logs to split. 2024-11-24T22:14:38,231 INFO [PEWorker-1 {}] master.SplitWALManager(105): bd6592c37bb2,41619,1732486469269 WAL count=0, meta=false 2024-11-24T22:14:38,231 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if bd6592c37bb2,41619,1732486469269 WAL splitting is done? wals=0, meta=false 2024-11-24T22:14:38,235 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN}, {pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN}, {pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN}, {pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f72c935213180e3ff989af795022b7e4, ASSIGN}, {pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=516ead6fe1773c4da8708cdc5713f526, ASSIGN}, {pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae, ASSIGN}, {pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=84dd509902edc43083bc78fe4b70922c, ASSIGN}, {pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4e91326d9778777879bc895ecb241af8, ASSIGN}] 2024-11-24T22:14:38,237 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4e91326d9778777879bc895ecb241af8, ASSIGN 2024-11-24T22:14:38,239 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4e91326d9778777879bc895ecb241af8, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-24T22:14:38,241 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=84dd509902edc43083bc78fe4b70922c, ASSIGN 2024-11-24T22:14:38,242 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=84dd509902edc43083bc78fe4b70922c, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-24T22:14:38,243 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae, ASSIGN 2024-11-24T22:14:38,244 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-24T22:14:38,247 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=516ead6fe1773c4da8708cdc5713f526, ASSIGN 2024-11-24T22:14:38,248 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=516ead6fe1773c4da8708cdc5713f526, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-24T22:14:38,250 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f72c935213180e3ff989af795022b7e4, ASSIGN 2024-11-24T22:14:38,251 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f72c935213180e3ff989af795022b7e4, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-24T22:14:38,255 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN 2024-11-24T22:14:38,257 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-24T22:14:38,259 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN 2024-11-24T22:14:38,260 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-24T22:14:38,261 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN 2024-11-24T22:14:38,262 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-24T22:14:38,390 DEBUG [bd6592c37bb2:36279 {}] balancer.BalancerClusterState(204): Hosts are {bd6592c37bb2=0} racks are {/default-rack=0} 2024-11-24T22:14:38,390 DEBUG [bd6592c37bb2:36279 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T22:14:38,390 DEBUG [bd6592c37bb2:36279 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T22:14:38,390 DEBUG [bd6592c37bb2:36279 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T22:14:38,390 DEBUG [bd6592c37bb2:36279 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T22:14:38,390 INFO [bd6592c37bb2:36279 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T22:14:38,390 INFO [bd6592c37bb2:36279 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T22:14:38,390 DEBUG [bd6592c37bb2:36279 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T22:14:38,391 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=516ead6fe1773c4da8708cdc5713f526, regionState=OPENING, regionLocation=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:38,392 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=73 updating hbase:meta row=84dd509902edc43083bc78fe4b70922c, regionState=OPENING, regionLocation=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:38,392 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=72 updating hbase:meta row=9b2e9ce57ffd0cd61c72602e5f6d9cae, regionState=OPENING, regionLocation=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:38,393 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=698dd9d64bdaf6a81e797d2590ff86f9, regionState=OPENING, regionLocation=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:38,393 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server bd6592c37bb2:41619 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bd6592c37bb2/172.17.0.2:41619 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:38,393 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=fc9f96fd4c44240a85e6b2c001e46153, regionState=OPENING, regionLocation=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:38,394 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=dc147c653b22684b8e41092cd6a0c8eb, regionState=OPENING, regionLocation=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:38,395 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=f72c935213180e3ff989af795022b7e4, regionState=OPENING, regionLocation=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:38,395 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,41619,1732486469269, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,41619,1732486469269, seqNum=-1, error=java.net.ConnectException: Call to address=bd6592c37bb2:41619 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bd6592c37bb2/172.17.0.2:41619 2024-11-24T22:14:38,395 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=74 updating hbase:meta row=4e91326d9778777879bc895ecb241af8, regionState=OPENING, regionLocation=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:38,395 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,41619,1732486469269, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-24T22:14:38,396 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,41619,1732486469269, seqNum=-1 from cache 2024-11-24T22:14:38,397 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address bd6592c37bb2:41619 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bd6592c37bb2/172.17.0.2:41619 2024-11-24T22:14:38,397 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server bd6592c37bb2:41619 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bd6592c37bb2/172.17.0.2:41619 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:38,398 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,41619,1732486469269, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=bd6592c37bb2:41619 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bd6592c37bb2/172.17.0.2:41619 2024-11-24T22:14:38,398 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,41619,1732486469269, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=bd6592c37bb2:41619 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bd6592c37bb2/172.17.0.2:41619 2024-11-24T22:14:38,398 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,41619,1732486469269, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=bd6592c37bb2:41619 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bd6592c37bb2/172.17.0.2:41619 2024-11-24T22:14:38,398 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,41619,1732486469269, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=bd6592c37bb2:41619 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bd6592c37bb2/172.17.0.2:41619 2024-11-24T22:14:38,398 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,41619,1732486469269, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=bd6592c37bb2:41619 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bd6592c37bb2/172.17.0.2:41619 2024-11-24T22:14:38,399 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,41619,1732486469269, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=bd6592c37bb2:41619 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bd6592c37bb2/172.17.0.2:41619 2024-11-24T22:14:38,399 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,41619,1732486469269, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=bd6592c37bb2:41619 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bd6592c37bb2/172.17.0.2:41619 2024-11-24T22:14:38,399 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address bd6592c37bb2:41619 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bd6592c37bb2/172.17.0.2:41619 2024-11-24T22:14:38,505 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T22:14:38,505 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bd6592c37bb2,42639,1732486469476, seqNum=-1] 2024-11-24T22:14:38,506 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T22:14:38,511 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39939, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T22:14:38,514 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=516ead6fe1773c4da8708cdc5713f526, ASSIGN because future has completed 2024-11-24T22:14:38,515 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 516ead6fe1773c4da8708cdc5713f526, server=bd6592c37bb2,38125,1732486469405}] 2024-11-24T22:14:38,517 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN because future has completed 2024-11-24T22:14:38,523 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN because future has completed 2024-11-24T22:14:38,526 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN because future has completed 2024-11-24T22:14:38,526 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure dc147c653b22684b8e41092cd6a0c8eb, server=bd6592c37bb2,42639,1732486469476}] 2024-11-24T22:14:38,528 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f72c935213180e3ff989af795022b7e4, ASSIGN because future has completed 2024-11-24T22:14:38,531 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4e91326d9778777879bc895ecb241af8, ASSIGN because future has completed 2024-11-24T22:14:38,536 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=84dd509902edc43083bc78fe4b70922c, ASSIGN because future has completed 2024-11-24T22:14:38,536 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405}] 2024-11-24T22:14:38,539 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae, ASSIGN because future has completed 2024-11-24T22:14:38,541 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9b2e9ce57ffd0cd61c72602e5f6d9cae, server=bd6592c37bb2,38125,1732486469405}] 2024-11-24T22:14:38,546 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e91326d9778777879bc895ecb241af8, server=bd6592c37bb2,42639,1732486469476}] 2024-11-24T22:14:38,551 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure f72c935213180e3ff989af795022b7e4, server=bd6592c37bb2,42639,1732486469476}] 2024-11-24T22:14:38,562 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure fc9f96fd4c44240a85e6b2c001e46153, server=bd6592c37bb2,42639,1732486469476}] 2024-11-24T22:14:38,566 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 698dd9d64bdaf6a81e797d2590ff86f9, server=bd6592c37bb2,38125,1732486469405}] 2024-11-24T22:14:38,676 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(132): Open test,sss,1732486472486.84dd509902edc43083bc78fe4b70922c. 2024-11-24T22:14:38,676 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7752): Opening region: {ENCODED => 84dd509902edc43083bc78fe4b70922c, NAME => 'test,sss,1732486472486.84dd509902edc43083bc78fe4b70922c.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-24T22:14:38,678 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 84dd509902edc43083bc78fe4b70922c 2024-11-24T22:14:38,678 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(898): Instantiated test,sss,1732486472486.84dd509902edc43083bc78fe4b70922c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:38,678 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7794): checking encryption for 84dd509902edc43083bc78fe4b70922c 2024-11-24T22:14:38,678 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7797): checking classloading for 84dd509902edc43083bc78fe4b70922c 2024-11-24T22:14:38,681 INFO [StoreOpener-84dd509902edc43083bc78fe4b70922c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 84dd509902edc43083bc78fe4b70922c 2024-11-24T22:14:38,682 INFO [StoreOpener-84dd509902edc43083bc78fe4b70922c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 84dd509902edc43083bc78fe4b70922c columnFamilyName family 2024-11-24T22:14:38,683 DEBUG [StoreOpener-84dd509902edc43083bc78fe4b70922c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:38,686 INFO [StoreOpener-84dd509902edc43083bc78fe4b70922c-1 {}] regionserver.HStore(327): Store=84dd509902edc43083bc78fe4b70922c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:14:38,686 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1038): replaying wal for 84dd509902edc43083bc78fe4b70922c 2024-11-24T22:14:38,687 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/84dd509902edc43083bc78fe4b70922c 2024-11-24T22:14:38,687 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(132): Open test,mmm,1732486472486.dc147c653b22684b8e41092cd6a0c8eb. 2024-11-24T22:14:38,687 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7752): Opening region: {ENCODED => dc147c653b22684b8e41092cd6a0c8eb, NAME => 'test,mmm,1732486472486.dc147c653b22684b8e41092cd6a0c8eb.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-24T22:14:38,688 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test dc147c653b22684b8e41092cd6a0c8eb 2024-11-24T22:14:38,688 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(898): Instantiated test,mmm,1732486472486.dc147c653b22684b8e41092cd6a0c8eb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:38,688 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7794): checking encryption for dc147c653b22684b8e41092cd6a0c8eb 2024-11-24T22:14:38,688 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7797): checking classloading for dc147c653b22684b8e41092cd6a0c8eb 2024-11-24T22:14:38,688 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/84dd509902edc43083bc78fe4b70922c 2024-11-24T22:14:38,689 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1048): stopping wal replay for 84dd509902edc43083bc78fe4b70922c 2024-11-24T22:14:38,689 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1060): Cleaning up temporary data for 84dd509902edc43083bc78fe4b70922c 2024-11-24T22:14:38,694 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1093): writing seq id for 84dd509902edc43083bc78fe4b70922c 2024-11-24T22:14:38,694 INFO [StoreOpener-dc147c653b22684b8e41092cd6a0c8eb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region dc147c653b22684b8e41092cd6a0c8eb 2024-11-24T22:14:38,696 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1114): Opened 84dd509902edc43083bc78fe4b70922c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62313174, jitterRate=-0.07146134972572327}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T22:14:38,696 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 84dd509902edc43083bc78fe4b70922c 2024-11-24T22:14:38,696 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1006): Region open journal for 84dd509902edc43083bc78fe4b70922c: Running coprocessor pre-open hook at 1732486478678Writing region info on filesystem at 1732486478678Initializing all the Stores at 1732486478680 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486478680Cleaning up temporary data from old regions at 1732486478689 (+9 ms)Running coprocessor post-open hooks at 1732486478696 (+7 ms)Region opened successfully at 1732486478696 2024-11-24T22:14:38,697 INFO [StoreOpener-dc147c653b22684b8e41092cd6a0c8eb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dc147c653b22684b8e41092cd6a0c8eb columnFamilyName family 2024-11-24T22:14:38,697 DEBUG [StoreOpener-dc147c653b22684b8e41092cd6a0c8eb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:38,698 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1732486472486.84dd509902edc43083bc78fe4b70922c., pid=77, masterSystemTime=1732486478670 2024-11-24T22:14:38,698 INFO [StoreOpener-dc147c653b22684b8e41092cd6a0c8eb-1 {}] regionserver.HStore(327): Store=dc147c653b22684b8e41092cd6a0c8eb/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:14:38,698 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1038): replaying wal for dc147c653b22684b8e41092cd6a0c8eb 2024-11-24T22:14:38,699 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/dc147c653b22684b8e41092cd6a0c8eb 2024-11-24T22:14:38,702 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1732486472486.84dd509902edc43083bc78fe4b70922c. 2024-11-24T22:14:38,702 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(153): Opened test,sss,1732486472486.84dd509902edc43083bc78fe4b70922c. 2024-11-24T22:14:38,702 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(132): Open test,jjj,1732486472486.9b2e9ce57ffd0cd61c72602e5f6d9cae. 2024-11-24T22:14:38,702 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7752): Opening region: {ENCODED => 9b2e9ce57ffd0cd61c72602e5f6d9cae, NAME => 'test,jjj,1732486472486.9b2e9ce57ffd0cd61c72602e5f6d9cae.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-24T22:14:38,703 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 9b2e9ce57ffd0cd61c72602e5f6d9cae 2024-11-24T22:14:38,703 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(898): Instantiated test,jjj,1732486472486.9b2e9ce57ffd0cd61c72602e5f6d9cae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:38,703 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7794): checking encryption for 9b2e9ce57ffd0cd61c72602e5f6d9cae 2024-11-24T22:14:38,703 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7797): checking classloading for 9b2e9ce57ffd0cd61c72602e5f6d9cae 2024-11-24T22:14:38,704 WARN [PEWorker-1 {}] procedure2.ProcedureTestingUtility(201): Set Kill before store update in rollback to: true 2024-11-24T22:14:38,704 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:38,704 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/dc147c653b22684b8e41092cd6a0c8eb 2024-11-24T22:14:38,706 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1048): stopping wal replay for dc147c653b22684b8e41092cd6a0c8eb 2024-11-24T22:14:38,706 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1060): Cleaning up temporary data for dc147c653b22684b8e41092cd6a0c8eb 2024-11-24T22:14:38,704 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1844): CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405 java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:38,709 INFO [StoreOpener-9b2e9ce57ffd0cd61c72602e5f6d9cae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 9b2e9ce57ffd0cd61c72602e5f6d9cae 2024-11-24T22:14:38,711 INFO [StoreOpener-9b2e9ce57ffd0cd61c72602e5f6d9cae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9b2e9ce57ffd0cd61c72602e5f6d9cae columnFamilyName family 2024-11-24T22:14:38,711 DEBUG [StoreOpener-9b2e9ce57ffd0cd61c72602e5f6d9cae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:38,717 INFO [StoreOpener-9b2e9ce57ffd0cd61c72602e5f6d9cae-1 {}] regionserver.HStore(327): Store=9b2e9ce57ffd0cd61c72602e5f6d9cae/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:14:38,717 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1038): replaying wal for 9b2e9ce57ffd0cd61c72602e5f6d9cae 2024-11-24T22:14:38,718 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/9b2e9ce57ffd0cd61c72602e5f6d9cae 2024-11-24T22:14:38,720 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/9b2e9ce57ffd0cd61c72602e5f6d9cae 2024-11-24T22:14:38,721 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1048): stopping wal replay for 9b2e9ce57ffd0cd61c72602e5f6d9cae 2024-11-24T22:14:38,721 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1060): Cleaning up temporary data for 9b2e9ce57ffd0cd61c72602e5f6d9cae 2024-11-24T22:14:38,722 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1093): writing seq id for dc147c653b22684b8e41092cd6a0c8eb 2024-11-24T22:14:38,721 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=true, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1845) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 4 more 2024-11-24T22:14:38,724 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1114): Opened dc147c653b22684b8e41092cd6a0c8eb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64817969, jitterRate=-0.034136995673179626}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T22:14:38,725 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1122): Running coprocessor post-open hooks for dc147c653b22684b8e41092cd6a0c8eb 2024-11-24T22:14:38,725 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1006): Region open journal for dc147c653b22684b8e41092cd6a0c8eb: Running coprocessor pre-open hook at 1732486478688Writing region info on filesystem at 1732486478688Initializing all the Stores at 1732486478692 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486478692Cleaning up temporary data from old regions at 1732486478706 (+14 ms)Running coprocessor post-open hooks at 1732486478725 (+19 ms)Region opened successfully at 1732486478725 2024-11-24T22:14:38,726 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1732486472486.dc147c653b22684b8e41092cd6a0c8eb., pid=76, masterSystemTime=1732486478681 2024-11-24T22:14:38,728 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1093): writing seq id for 9b2e9ce57ffd0cd61c72602e5f6d9cae 2024-11-24T22:14:38,729 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1732486472486.dc147c653b22684b8e41092cd6a0c8eb. 2024-11-24T22:14:38,729 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(153): Opened test,mmm,1732486472486.dc147c653b22684b8e41092cd6a0c8eb. 2024-11-24T22:14:38,729 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(132): Open test,ggg,1732486472486.f72c935213180e3ff989af795022b7e4. 2024-11-24T22:14:38,729 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7752): Opening region: {ENCODED => f72c935213180e3ff989af795022b7e4, NAME => 'test,ggg,1732486472486.f72c935213180e3ff989af795022b7e4.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-24T22:14:38,729 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f72c935213180e3ff989af795022b7e4 2024-11-24T22:14:38,730 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(898): Instantiated test,ggg,1732486472486.f72c935213180e3ff989af795022b7e4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:38,730 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7794): checking encryption for f72c935213180e3ff989af795022b7e4 2024-11-24T22:14:38,730 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7797): checking classloading for f72c935213180e3ff989af795022b7e4 2024-11-24T22:14:38,731 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1114): Opened 9b2e9ce57ffd0cd61c72602e5f6d9cae; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66929142, jitterRate=-0.0026780664920806885}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T22:14:38,731 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9b2e9ce57ffd0cd61c72602e5f6d9cae 2024-11-24T22:14:38,731 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1006): Region open journal for 9b2e9ce57ffd0cd61c72602e5f6d9cae: Running coprocessor pre-open hook at 1732486478703Writing region info on filesystem at 1732486478703Initializing all the Stores at 1732486478706 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486478706Cleaning up temporary data from old regions at 1732486478721 (+15 ms)Running coprocessor post-open hooks at 1732486478731 (+10 ms)Region opened successfully at 1732486478731 2024-11-24T22:14:38,732 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2236): Post open deploy tasks for test,jjj,1732486472486.9b2e9ce57ffd0cd61c72602e5f6d9cae., pid=78, masterSystemTime=1732486478670 2024-11-24T22:14:38,733 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T22:14:38,734 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=82, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 698dd9d64bdaf6a81e797d2590ff86f9, server=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:38,734 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T22:14:38,734 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=82, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 698dd9d64bdaf6a81e797d2590ff86f9, server=bd6592c37bb2,38125,1732486469405 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:38,735 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2266): Finished post open deploy task for test,jjj,1732486472486.9b2e9ce57ffd0cd61c72602e5f6d9cae. 2024-11-24T22:14:38,735 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(153): Opened test,jjj,1732486472486.9b2e9ce57ffd0cd61c72602e5f6d9cae. 2024-11-24T22:14:38,735 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(132): Open test,xxx,1732486472486.516ead6fe1773c4da8708cdc5713f526. 2024-11-24T22:14:38,735 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7752): Opening region: {ENCODED => 516ead6fe1773c4da8708cdc5713f526, NAME => 'test,xxx,1732486472486.516ead6fe1773c4da8708cdc5713f526.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-11-24T22:14:38,736 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 516ead6fe1773c4da8708cdc5713f526 2024-11-24T22:14:38,736 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(898): Instantiated test,xxx,1732486472486.516ead6fe1773c4da8708cdc5713f526.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:38,736 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7794): checking encryption for 516ead6fe1773c4da8708cdc5713f526 2024-11-24T22:14:38,736 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7797): checking classloading for 516ead6fe1773c4da8708cdc5713f526 2024-11-24T22:14:38,741 INFO [StoreOpener-f72c935213180e3ff989af795022b7e4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f72c935213180e3ff989af795022b7e4 2024-11-24T22:14:38,745 INFO [StoreOpener-f72c935213180e3ff989af795022b7e4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f72c935213180e3ff989af795022b7e4 columnFamilyName family 2024-11-24T22:14:38,745 DEBUG [StoreOpener-f72c935213180e3ff989af795022b7e4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:38,746 INFO [StoreOpener-f72c935213180e3ff989af795022b7e4-1 {}] regionserver.HStore(327): Store=f72c935213180e3ff989af795022b7e4/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:14:38,746 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1038): replaying wal for f72c935213180e3ff989af795022b7e4 2024-11-24T22:14:38,747 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/f72c935213180e3ff989af795022b7e4 2024-11-24T22:14:38,749 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/f72c935213180e3ff989af795022b7e4 2024-11-24T22:14:38,751 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1048): stopping wal replay for f72c935213180e3ff989af795022b7e4 2024-11-24T22:14:38,751 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1060): Cleaning up temporary data for f72c935213180e3ff989af795022b7e4 2024-11-24T22:14:38,755 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1093): writing seq id for f72c935213180e3ff989af795022b7e4 2024-11-24T22:14:38,758 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1114): Opened f72c935213180e3ff989af795022b7e4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60139490, jitterRate=-0.10385176539421082}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T22:14:38,758 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f72c935213180e3ff989af795022b7e4 2024-11-24T22:14:38,758 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1006): Region open journal for f72c935213180e3ff989af795022b7e4: Running coprocessor pre-open hook at 1732486478730Writing region info on filesystem at 1732486478730Initializing all the Stores at 1732486478731 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486478731Cleaning up temporary data from old regions at 1732486478751 (+20 ms)Running coprocessor post-open hooks at 1732486478758 (+7 ms)Region opened successfully at 1732486478758 2024-11-24T22:14:38,760 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ggg,1732486472486.f72c935213180e3ff989af795022b7e4., pid=80, masterSystemTime=1732486478681 2024-11-24T22:14:38,761 INFO [StoreOpener-516ead6fe1773c4da8708cdc5713f526-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 516ead6fe1773c4da8708cdc5713f526 2024-11-24T22:14:38,763 INFO [StoreOpener-516ead6fe1773c4da8708cdc5713f526-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 516ead6fe1773c4da8708cdc5713f526 columnFamilyName family 2024-11-24T22:14:38,763 DEBUG [StoreOpener-516ead6fe1773c4da8708cdc5713f526-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:38,764 INFO [StoreOpener-516ead6fe1773c4da8708cdc5713f526-1 {}] regionserver.HStore(327): Store=516ead6fe1773c4da8708cdc5713f526/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:14:38,764 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1038): replaying wal for 516ead6fe1773c4da8708cdc5713f526 2024-11-24T22:14:38,765 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/516ead6fe1773c4da8708cdc5713f526 2024-11-24T22:14:38,767 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ggg,1732486472486.f72c935213180e3ff989af795022b7e4. 2024-11-24T22:14:38,767 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(153): Opened test,ggg,1732486472486.f72c935213180e3ff989af795022b7e4. 2024-11-24T22:14:38,767 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(132): Open test,vvv,1732486472486.fc9f96fd4c44240a85e6b2c001e46153. 2024-11-24T22:14:38,767 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7752): Opening region: {ENCODED => fc9f96fd4c44240a85e6b2c001e46153, NAME => 'test,vvv,1732486472486.fc9f96fd4c44240a85e6b2c001e46153.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-24T22:14:38,767 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test fc9f96fd4c44240a85e6b2c001e46153 2024-11-24T22:14:38,768 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(898): Instantiated test,vvv,1732486472486.fc9f96fd4c44240a85e6b2c001e46153.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:38,768 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7794): checking encryption for fc9f96fd4c44240a85e6b2c001e46153 2024-11-24T22:14:38,768 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7797): checking classloading for fc9f96fd4c44240a85e6b2c001e46153 2024-11-24T22:14:38,770 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/516ead6fe1773c4da8708cdc5713f526 2024-11-24T22:14:38,771 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1048): stopping wal replay for 516ead6fe1773c4da8708cdc5713f526 2024-11-24T22:14:38,771 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1060): Cleaning up temporary data for 516ead6fe1773c4da8708cdc5713f526 2024-11-24T22:14:38,774 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): writing seq id for 516ead6fe1773c4da8708cdc5713f526 2024-11-24T22:14:38,775 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1114): Opened 516ead6fe1773c4da8708cdc5713f526; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66267813, jitterRate=-0.012532636523246765}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T22:14:38,775 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 516ead6fe1773c4da8708cdc5713f526 2024-11-24T22:14:38,775 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1006): Region open journal for 516ead6fe1773c4da8708cdc5713f526: Running coprocessor pre-open hook at 1732486478736Writing region info on filesystem at 1732486478736Initializing all the Stores at 1732486478737 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486478737Cleaning up temporary data from old regions at 1732486478771 (+34 ms)Running coprocessor post-open hooks at 1732486478775 (+4 ms)Region opened successfully at 1732486478775 2024-11-24T22:14:38,776 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1732486472486.516ead6fe1773c4da8708cdc5713f526., pid=75, masterSystemTime=1732486478670 2024-11-24T22:14:38,777 INFO [StoreOpener-fc9f96fd4c44240a85e6b2c001e46153-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region fc9f96fd4c44240a85e6b2c001e46153 2024-11-24T22:14:38,779 INFO [StoreOpener-fc9f96fd4c44240a85e6b2c001e46153-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fc9f96fd4c44240a85e6b2c001e46153 columnFamilyName family 2024-11-24T22:14:38,779 DEBUG [StoreOpener-fc9f96fd4c44240a85e6b2c001e46153-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:38,780 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1732486472486.516ead6fe1773c4da8708cdc5713f526. 2024-11-24T22:14:38,781 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(153): Opened test,xxx,1732486472486.516ead6fe1773c4da8708cdc5713f526. 2024-11-24T22:14:38,781 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] handler.AssignRegionHandler(132): Open test,ddd,1732486472486.698dd9d64bdaf6a81e797d2590ff86f9. 2024-11-24T22:14:38,781 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7752): Opening region: {ENCODED => 698dd9d64bdaf6a81e797d2590ff86f9, NAME => 'test,ddd,1732486472486.698dd9d64bdaf6a81e797d2590ff86f9.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-11-24T22:14:38,781 INFO [StoreOpener-fc9f96fd4c44240a85e6b2c001e46153-1 {}] regionserver.HStore(327): Store=fc9f96fd4c44240a85e6b2c001e46153/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:14:38,781 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 698dd9d64bdaf6a81e797d2590ff86f9 2024-11-24T22:14:38,781 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(898): Instantiated test,ddd,1732486472486.698dd9d64bdaf6a81e797d2590ff86f9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:38,781 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1038): replaying wal for fc9f96fd4c44240a85e6b2c001e46153 2024-11-24T22:14:38,782 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7794): checking encryption for 698dd9d64bdaf6a81e797d2590ff86f9 2024-11-24T22:14:38,782 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7797): checking classloading for 698dd9d64bdaf6a81e797d2590ff86f9 2024-11-24T22:14:38,783 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/fc9f96fd4c44240a85e6b2c001e46153 2024-11-24T22:14:38,784 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/fc9f96fd4c44240a85e6b2c001e46153 2024-11-24T22:14:38,785 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1048): stopping wal replay for fc9f96fd4c44240a85e6b2c001e46153 2024-11-24T22:14:38,785 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1060): Cleaning up temporary data for fc9f96fd4c44240a85e6b2c001e46153 2024-11-24T22:14:38,787 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1093): writing seq id for fc9f96fd4c44240a85e6b2c001e46153 2024-11-24T22:14:38,789 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1114): Opened fc9f96fd4c44240a85e6b2c001e46153; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65230816, jitterRate=-0.027985095977783203}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T22:14:38,789 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fc9f96fd4c44240a85e6b2c001e46153 2024-11-24T22:14:38,789 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1006): Region open journal for fc9f96fd4c44240a85e6b2c001e46153: Running coprocessor pre-open hook at 1732486478768Writing region info on filesystem at 1732486478768Initializing all the Stores at 1732486478769 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486478769Cleaning up temporary data from old regions at 1732486478785 (+16 ms)Running coprocessor post-open hooks at 1732486478789 (+4 ms)Region opened successfully at 1732486478789 2024-11-24T22:14:38,790 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1732486472486.fc9f96fd4c44240a85e6b2c001e46153., pid=81, masterSystemTime=1732486478681 2024-11-24T22:14:38,795 INFO [StoreOpener-698dd9d64bdaf6a81e797d2590ff86f9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 698dd9d64bdaf6a81e797d2590ff86f9 2024-11-24T22:14:38,796 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1732486472486.fc9f96fd4c44240a85e6b2c001e46153. 2024-11-24T22:14:38,796 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(153): Opened test,vvv,1732486472486.fc9f96fd4c44240a85e6b2c001e46153. 2024-11-24T22:14:38,796 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open test,,1732486472486.4e91326d9778777879bc895ecb241af8. 2024-11-24T22:14:38,797 INFO [StoreOpener-698dd9d64bdaf6a81e797d2590ff86f9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 698dd9d64bdaf6a81e797d2590ff86f9 columnFamilyName family 2024-11-24T22:14:38,797 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => 4e91326d9778777879bc895ecb241af8, NAME => 'test,,1732486472486.4e91326d9778777879bc895ecb241af8.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-24T22:14:38,797 DEBUG [StoreOpener-698dd9d64bdaf6a81e797d2590ff86f9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:38,797 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4e91326d9778777879bc895ecb241af8 2024-11-24T22:14:38,797 INFO [StoreOpener-698dd9d64bdaf6a81e797d2590ff86f9-1 {}] regionserver.HStore(327): Store=698dd9d64bdaf6a81e797d2590ff86f9/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:14:38,797 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated test,,1732486472486.4e91326d9778777879bc895ecb241af8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T22:14:38,798 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for 4e91326d9778777879bc895ecb241af8 2024-11-24T22:14:38,798 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1038): replaying wal for 698dd9d64bdaf6a81e797d2590ff86f9 2024-11-24T22:14:38,798 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for 4e91326d9778777879bc895ecb241af8 2024-11-24T22:14:38,798 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/698dd9d64bdaf6a81e797d2590ff86f9 2024-11-24T22:14:38,799 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T22:14:38,799 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T22:14:38,799 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T22:14:38,799 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T22:14:38,802 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36279 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:38,804 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/698dd9d64bdaf6a81e797d2590ff86f9 2024-11-24T22:14:38,804 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1048): stopping wal replay for 698dd9d64bdaf6a81e797d2590ff86f9 2024-11-24T22:14:38,804 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1060): Cleaning up temporary data for 698dd9d64bdaf6a81e797d2590ff86f9 2024-11-24T22:14:38,807 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1093): writing seq id for 698dd9d64bdaf6a81e797d2590ff86f9 2024-11-24T22:14:38,809 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T22:14:38,809 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T22:14:38,809 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T22:14:38,810 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T22:14:38,814 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 3 msec 2024-11-24T22:14:38,815 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1114): Opened 698dd9d64bdaf6a81e797d2590ff86f9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67882899, jitterRate=0.01153402030467987}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T22:14:38,815 INFO [StoreOpener-4e91326d9778777879bc895ecb241af8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4e91326d9778777879bc895ecb241af8 2024-11-24T22:14:38,817 INFO [StoreOpener-4e91326d9778777879bc895ecb241af8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4e91326d9778777879bc895ecb241af8 columnFamilyName family 2024-11-24T22:14:38,817 DEBUG [StoreOpener-4e91326d9778777879bc895ecb241af8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T22:14:38,818 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 698dd9d64bdaf6a81e797d2590ff86f9 2024-11-24T22:14:38,818 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1006): Region open journal for 698dd9d64bdaf6a81e797d2590ff86f9: Running coprocessor pre-open hook at 1732486478782Writing region info on filesystem at 1732486478782Initializing all the Stores at 1732486478783 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486478783Cleaning up temporary data from old regions at 1732486478804 (+21 ms)Running coprocessor post-open hooks at 1732486478818 (+14 ms)Region opened successfully at 1732486478818 2024-11-24T22:14:38,819 INFO [StoreOpener-4e91326d9778777879bc895ecb241af8-1 {}] regionserver.HStore(327): Store=4e91326d9778777879bc895ecb241af8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T22:14:38,819 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for 4e91326d9778777879bc895ecb241af8 2024-11-24T22:14:38,820 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/4e91326d9778777879bc895ecb241af8 2024-11-24T22:14:38,821 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ddd,1732486472486.698dd9d64bdaf6a81e797d2590ff86f9., pid=82, masterSystemTime=1732486478670 2024-11-24T22:14:38,822 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/4e91326d9778777879bc895ecb241af8 2024-11-24T22:14:38,822 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36279 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:38,822 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for 4e91326d9778777879bc895ecb241af8 2024-11-24T22:14:38,822 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T22:14:38,822 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for 4e91326d9778777879bc895ecb241af8 2024-11-24T22:14:38,823 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T22:14:38,823 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T22:14:38,823 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:38,823 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T22:14:38,823 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:38,823 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=81, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T22:14:38,823 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:38,823 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=82, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T22:14:38,823 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:38,823 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T22:14:38,823 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:38,823 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T22:14:38,823 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:38,824 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T22:14:38,824 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:38,824 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T22:14:38,824 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:38,824 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T22:14:38,824 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T22:14:38,824 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T22:14:38,824 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:38,824 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T22:14:38,823 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "bd6592c37bb2" port: 38125 start_code: 1732486469405 } transition { transition_code: OPENED region_info { region_id: 1732486472486 table_name { namespace: "default" qualifier: "test" } start_key: "ddd" end_key: "eee" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 82 initiating_master_active_time: 1732486469596 }; retry (#0) after 1009ms delay (Master is coming online...). org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-24T22:14:38,825 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T22:14:38,825 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for 4e91326d9778777879bc895ecb241af8 2024-11-24T22:14:38,826 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened 4e91326d9778777879bc895ecb241af8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60541720, jitterRate=-0.09785807132720947}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T22:14:38,826 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4e91326d9778777879bc895ecb241af8 2024-11-24T22:14:38,826 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for 4e91326d9778777879bc895ecb241af8: Running coprocessor pre-open hook at 1732486478798Writing region info on filesystem at 1732486478798Initializing all the Stores at 1732486478802 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732486478802Cleaning up temporary data from old regions at 1732486478822 (+20 ms)Running coprocessor post-open hooks at 1732486478826 (+4 ms)Region opened successfully at 1732486478826 2024-11-24T22:14:38,827 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T22:14:38,827 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=cca6a66154e5c299fad3a247afbfb5fd, REOPEN/MOVE 2024-11-24T22:14:38,827 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1732486472486.4e91326d9778777879bc895ecb241af8., pid=79, masterSystemTime=1732486478681 2024-11-24T22:14:38,829 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36279 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:38,830 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "bd6592c37bb2" port: 42639 start_code: 1732486469476 } transition { transition_code: OPENED region_info { region_id: 1732486472486 table_name { namespace: "default" qualifier: "test" } start_key: "" end_key: "bbb" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 79 initiating_master_active_time: 1732486469596 }; retry (#0) after 1008ms delay (Master is coming online...). org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-24T22:14:38,832 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true 2024-11-24T22:14:38,833 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:38,833 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:38,833 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T22:14:38,833 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:38,833 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:38,833 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:38,833 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN 2024-11-24T22:14:38,833 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN 2024-11-24T22:14:38,833 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN 2024-11-24T22:14:38,833 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f72c935213180e3ff989af795022b7e4, ASSIGN 2024-11-24T22:14:38,833 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=516ead6fe1773c4da8708cdc5713f526, ASSIGN 2024-11-24T22:14:38,833 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae, ASSIGN 2024-11-24T22:14:38,833 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=84dd509902edc43083bc78fe4b70922c, ASSIGN 2024-11-24T22:14:38,833 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e91326d9778777879bc895ecb241af8, ASSIGN 2024-11-24T22:14:38,833 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 516ead6fe1773c4da8708cdc5713f526, server=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:38,833 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure dc147c653b22684b8e41092cd6a0c8eb, server=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:38,833 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=73, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:38,833 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9b2e9ce57ffd0cd61c72602e5f6d9cae, server=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:38,833 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e91326d9778777879bc895ecb241af8, server=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:38,833 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure f72c935213180e3ff989af795022b7e4, server=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:38,833 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=81, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure fc9f96fd4c44240a85e6b2c001e46153, server=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:38,833 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=82, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 698dd9d64bdaf6a81e797d2590ff86f9, server=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:38,833 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:38,833 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:38,834 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:38,834 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:38,834 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:38,834 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:38,834 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:38,834 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:38,834 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN 2024-11-24T22:14:38,834 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:38,834 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN 2024-11-24T22:14:38,834 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:38,834 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN 2024-11-24T22:14:38,834 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f72c935213180e3ff989af795022b7e4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:38,834 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f72c935213180e3ff989af795022b7e4, ASSIGN 2024-11-24T22:14:38,834 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=516ead6fe1773c4da8708cdc5713f526, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:38,834 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=516ead6fe1773c4da8708cdc5713f526, ASSIGN 2024-11-24T22:14:38,834 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:38,834 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae, ASSIGN 2024-11-24T22:14:38,834 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=84dd509902edc43083bc78fe4b70922c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:38,834 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=84dd509902edc43083bc78fe4b70922c, ASSIGN 2024-11-24T22:14:38,834 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e91326d9778777879bc895ecb241af8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:38,834 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e91326d9778777879bc895ecb241af8, ASSIGN 2024-11-24T22:14:38,834 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 516ead6fe1773c4da8708cdc5713f526, server=bd6592c37bb2,38125,1732486469405 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:38,834 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure dc147c653b22684b8e41092cd6a0c8eb, server=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:38,834 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=73, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:38,834 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9b2e9ce57ffd0cd61c72602e5f6d9cae, server=bd6592c37bb2,38125,1732486469405 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:38,834 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e91326d9778777879bc895ecb241af8, server=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:38,834 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure f72c935213180e3ff989af795022b7e4, server=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:38,834 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=81, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure fc9f96fd4c44240a85e6b2c001e46153, server=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:38,835 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=82, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 698dd9d64bdaf6a81e797d2590ff86f9, server=bd6592c37bb2,38125,1732486469405 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:38,835 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 21 msec 2024-11-24T22:14:38,839 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=65 2024-11-24T22:14:38,839 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=bd6592c37bb2,42639,1732486469476, table=hbase:meta, region=1588230740 2024-11-24T22:14:38,839 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bd6592c37bb2,42639,1732486469476, state=OPEN 2024-11-24T22:14:38,842 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42639-0x1014c0b79ef0003, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:38,842 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38125-0x1014c0b79ef0002, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:38,842 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:38,842 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:38,842 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-24T22:14:38,843 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-24T22:14:38,844 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test 2024-11-24T22:14:38,844 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36279-0x1014c0b79ef0000, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:38,844 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test Metrics about Tables on a single HBase RegionServer 2024-11-24T22:14:38,844 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:38,845 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T22:14:38,846 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-24T22:14:38,846 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-24T22:14:38,846 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-24T22:14:38,849 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN to state=OFFLINE, location=null, table=test, region=dc147c653b22684b8e41092cd6a0c8eb to restore RIT 2024-11-24T22:14:38,849 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN to state=OFFLINE, location=null, table=test, region=fc9f96fd4c44240a85e6b2c001e46153 to restore RIT 2024-11-24T22:14:38,849 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN to state=OFFLINE, location=null, table=test, region=698dd9d64bdaf6a81e797d2590ff86f9 to restore RIT 2024-11-24T22:14:38,849 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f72c935213180e3ff989af795022b7e4, ASSIGN to state=OFFLINE, location=null, table=test, region=f72c935213180e3ff989af795022b7e4 to restore RIT 2024-11-24T22:14:38,849 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=516ead6fe1773c4da8708cdc5713f526, ASSIGN to state=OFFLINE, location=null, table=test, region=516ead6fe1773c4da8708cdc5713f526 to restore RIT 2024-11-24T22:14:38,849 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae, ASSIGN to state=OFFLINE, location=null, table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae to restore RIT 2024-11-24T22:14:38,849 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=84dd509902edc43083bc78fe4b70922c, ASSIGN to state=OFFLINE, location=null, table=test, region=84dd509902edc43083bc78fe4b70922c to restore RIT 2024-11-24T22:14:38,849 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e91326d9778777879bc895ecb241af8, ASSIGN to state=OFFLINE, location=null, table=test, region=4e91326d9778777879bc895ecb241af8 to restore RIT 2024-11-24T22:14:38,869 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732486508869 2024-11-24T22:14:38,870 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T22:14:38,871 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:38,877 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T22:14:38,881 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T22:14:38,882 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=81, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure fc9f96fd4c44240a85e6b2c001e46153, server=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:38,882 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T22:14:38,882 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=81, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure fc9f96fd4c44240a85e6b2c001e46153, server=bd6592c37bb2,42639,1732486469476 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:38,924 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e91326d9778777879bc895ecb241af8, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:38,924 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4279880609a62ccb9e9fb29e79bdf7e7, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:38,925 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c68a44a11eeaccf8a3c2f49ba1446c1e, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:38,925 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=698dd9d64bdaf6a81e797d2590ff86f9, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:38,925 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46b79f5a3db63445091682b5e72f635a, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:38,926 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=736f44b8f348e2a5a96dfd3abc5e62b2, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:38,926 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f72c935213180e3ff989af795022b7e4, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:38,926 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=72a41015f7b88194cc3a08feb66aa4f9, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:38,927 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94f2fa48145fe3bdeffbfded6a4edd73, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:38,927 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b2e9ce57ffd0cd61c72602e5f6d9cae, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:38,928 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b86957c7b81466a818bbff45fac751f, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:38,928 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=967acf99df098c28d8f837a9f0746a90, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:38,928 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc147c653b22684b8e41092cd6a0c8eb, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:38,930 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5160ff1dd0e56b08fbca2ffe581141d4, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:38,930 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce1f0bcb3026cb1cbbd9e63c21b83f99, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:38,931 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cca6a66154e5c299fad3a247afbfb5fd, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=5 2024-11-24T22:14:38,931 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83e58b0e0989e11c83283bf7bd41306c, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:38,931 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c802d82f160c5269119d007388afb85e, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:38,932 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=84dd509902edc43083bc78fe4b70922c, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:38,932 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acdab7e426c93b10b34b4d331b21cef0, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:38,932 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79c2dd6f654fabefe0fc06b8199e8562, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:38,933 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc9f96fd4c44240a85e6b2c001e46153, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:38,933 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50d9b38af2d6413e500860133d150aa8, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:38,933 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=516ead6fe1773c4da8708cdc5713f526, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:38,934 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79d315a497ec3ad891988cb800bae337, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:38,934 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed5b70c013e388180062b1513b980a07, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:38,934 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T22:14:38,934 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732486538934 2024-11-24T22:14:38,934 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732486598934 2024-11-24T22:14:38,934 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 64 msec 2024-11-24T22:14:38,934 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T22:14:38,934 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T22:14:38,935 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T22:14:38,935 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T22:14:38,937 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T22:14:38,937 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T22:14:38,937 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T22:14:38,940 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T22:14:38,941 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-24T22:14:38,950 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T22:14:38,950 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T22:14:38,950 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T22:14:38,950 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:38,950 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T22:14:38,950 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:38,950 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=81, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T22:14:38,950 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:38,950 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:38,950 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T22:14:38,950 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:38,950 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T22:14:38,950 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:38,950 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T22:14:38,950 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:38,950 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T22:14:38,950 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:38,951 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T22:14:38,951 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T22:14:38,951 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T22:14:38,951 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:38,951 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T22:14:38,951 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T22:14:38,952 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T22:14:38,953 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=cca6a66154e5c299fad3a247afbfb5fd, REOPEN/MOVE 2024-11-24T22:14:38,956 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true 2024-11-24T22:14:38,956 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:38,957 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:38,957 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T22:14:38,957 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:38,957 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:38,957 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:38,957 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN 2024-11-24T22:14:38,957 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN 2024-11-24T22:14:38,957 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN 2024-11-24T22:14:38,957 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f72c935213180e3ff989af795022b7e4, ASSIGN 2024-11-24T22:14:38,957 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=516ead6fe1773c4da8708cdc5713f526, ASSIGN 2024-11-24T22:14:38,957 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae, ASSIGN 2024-11-24T22:14:38,957 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=84dd509902edc43083bc78fe4b70922c, ASSIGN 2024-11-24T22:14:38,957 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e91326d9778777879bc895ecb241af8, ASSIGN 2024-11-24T22:14:38,957 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 516ead6fe1773c4da8708cdc5713f526, server=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:38,957 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure dc147c653b22684b8e41092cd6a0c8eb, server=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:38,957 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=73, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:38,957 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9b2e9ce57ffd0cd61c72602e5f6d9cae, server=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:38,957 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e91326d9778777879bc895ecb241af8, server=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:38,957 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure f72c935213180e3ff989af795022b7e4, server=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:38,957 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=81, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure fc9f96fd4c44240a85e6b2c001e46153, server=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:38,957 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:38,957 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:38,957 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:38,958 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:38,958 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:38,958 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:38,958 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:38,958 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:38,958 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN 2024-11-24T22:14:38,958 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:38,958 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN 2024-11-24T22:14:38,958 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:38,958 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN 2024-11-24T22:14:38,958 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f72c935213180e3ff989af795022b7e4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:38,958 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f72c935213180e3ff989af795022b7e4, ASSIGN 2024-11-24T22:14:38,958 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=516ead6fe1773c4da8708cdc5713f526, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:38,958 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=516ead6fe1773c4da8708cdc5713f526, ASSIGN 2024-11-24T22:14:38,958 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:38,958 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae, ASSIGN 2024-11-24T22:14:38,958 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=84dd509902edc43083bc78fe4b70922c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:38,958 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=84dd509902edc43083bc78fe4b70922c, ASSIGN 2024-11-24T22:14:38,958 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e91326d9778777879bc895ecb241af8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:38,958 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e91326d9778777879bc895ecb241af8, ASSIGN 2024-11-24T22:14:38,958 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 516ead6fe1773c4da8708cdc5713f526, server=bd6592c37bb2,38125,1732486469405 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:38,958 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure dc147c653b22684b8e41092cd6a0c8eb, server=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:38,958 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=73, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:38,958 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9b2e9ce57ffd0cd61c72602e5f6d9cae, server=bd6592c37bb2,38125,1732486469405 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:38,958 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e91326d9778777879bc895ecb241af8, server=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:38,958 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure f72c935213180e3ff989af795022b7e4, server=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:38,958 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=81, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure fc9f96fd4c44240a85e6b2c001e46153, server=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:38,959 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 17 msec 2024-11-24T22:14:38,968 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=65 2024-11-24T22:14:38,968 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=bd6592c37bb2,42639,1732486469476, table=hbase:meta, region=1588230740 2024-11-24T22:14:38,968 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bd6592c37bb2,42639,1732486469476, state=OPEN 2024-11-24T22:14:38,972 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42639-0x1014c0b79ef0003, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:38,972 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38125-0x1014c0b79ef0002, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:38,972 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36279-0x1014c0b79ef0000, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:38,972 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:38,972 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:38,972 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:38,976 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN to state=OFFLINE, location=null, table=test, region=dc147c653b22684b8e41092cd6a0c8eb to restore RIT 2024-11-24T22:14:38,976 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN to state=OFFLINE, location=null, table=test, region=fc9f96fd4c44240a85e6b2c001e46153 to restore RIT 2024-11-24T22:14:38,976 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN to state=OFFLINE, location=null, table=test, region=698dd9d64bdaf6a81e797d2590ff86f9 to restore RIT 2024-11-24T22:14:38,976 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f72c935213180e3ff989af795022b7e4, ASSIGN to state=OFFLINE, location=null, table=test, region=f72c935213180e3ff989af795022b7e4 to restore RIT 2024-11-24T22:14:38,976 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=516ead6fe1773c4da8708cdc5713f526, ASSIGN to state=OFFLINE, location=null, table=test, region=516ead6fe1773c4da8708cdc5713f526 to restore RIT 2024-11-24T22:14:38,976 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae, ASSIGN to state=OFFLINE, location=null, table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae to restore RIT 2024-11-24T22:14:38,976 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=84dd509902edc43083bc78fe4b70922c, ASSIGN to state=OFFLINE, location=null, table=test, region=84dd509902edc43083bc78fe4b70922c to restore RIT 2024-11-24T22:14:38,977 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e91326d9778777879bc895ecb241af8, ASSIGN to state=OFFLINE, location=null, table=test, region=4e91326d9778777879bc895ecb241af8 to restore RIT 2024-11-24T22:14:39,007 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732486509007 2024-11-24T22:14:39,007 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T22:14:39,020 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e91326d9778777879bc895ecb241af8, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,020 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4279880609a62ccb9e9fb29e79bdf7e7, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,009 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:39,022 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T22:14:39,024 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c68a44a11eeaccf8a3c2f49ba1446c1e, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,025 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=698dd9d64bdaf6a81e797d2590ff86f9, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,025 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46b79f5a3db63445091682b5e72f635a, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,025 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=736f44b8f348e2a5a96dfd3abc5e62b2, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,025 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f72c935213180e3ff989af795022b7e4, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,026 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=72a41015f7b88194cc3a08feb66aa4f9, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,026 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94f2fa48145fe3bdeffbfded6a4edd73, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,026 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b2e9ce57ffd0cd61c72602e5f6d9cae, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,026 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b86957c7b81466a818bbff45fac751f, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,027 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=967acf99df098c28d8f837a9f0746a90, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,027 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T22:14:39,027 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=80, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure f72c935213180e3ff989af795022b7e4, server=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:39,027 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T22:14:39,027 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc147c653b22684b8e41092cd6a0c8eb, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,027 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5160ff1dd0e56b08fbca2ffe581141d4, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,027 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=80, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure f72c935213180e3ff989af795022b7e4, server=bd6592c37bb2,42639,1732486469476 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:39,027 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce1f0bcb3026cb1cbbd9e63c21b83f99, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,028 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cca6a66154e5c299fad3a247afbfb5fd, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=5 2024-11-24T22:14:39,028 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83e58b0e0989e11c83283bf7bd41306c, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,028 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c802d82f160c5269119d007388afb85e, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,028 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=84dd509902edc43083bc78fe4b70922c, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,029 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acdab7e426c93b10b34b4d331b21cef0, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,029 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79c2dd6f654fabefe0fc06b8199e8562, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,029 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc9f96fd4c44240a85e6b2c001e46153, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,029 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50d9b38af2d6413e500860133d150aa8, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,030 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=516ead6fe1773c4da8708cdc5713f526, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,030 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79d315a497ec3ad891988cb800bae337, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,030 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed5b70c013e388180062b1513b980a07, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,030 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T22:14:39,030 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732486539030 2024-11-24T22:14:39,030 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732486599030 2024-11-24T22:14:39,030 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 22 msec 2024-11-24T22:14:39,030 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T22:14:39,030 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T22:14:39,031 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T22:14:39,031 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T22:14:39,031 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T22:14:39,032 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T22:14:39,032 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T22:14:39,032 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T22:14:39,038 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 5 msec 2024-11-24T22:14:39,044 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T22:14:39,044 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T22:14:39,044 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T22:14:39,044 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:39,044 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T22:14:39,044 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:39,044 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:39,044 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:39,044 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T22:14:39,044 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:39,044 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T22:14:39,044 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:39,044 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T22:14:39,044 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:39,044 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T22:14:39,044 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:39,044 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T22:14:39,044 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T22:14:39,044 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T22:14:39,044 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:39,044 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T22:14:39,045 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T22:14:39,046 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T22:14:39,046 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=cca6a66154e5c299fad3a247afbfb5fd, REOPEN/MOVE 2024-11-24T22:14:39,053 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true 2024-11-24T22:14:39,053 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:39,053 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:39,054 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T22:14:39,054 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:39,054 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:39,054 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:39,054 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN 2024-11-24T22:14:39,054 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN 2024-11-24T22:14:39,054 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN 2024-11-24T22:14:39,054 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f72c935213180e3ff989af795022b7e4, ASSIGN 2024-11-24T22:14:39,054 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=516ead6fe1773c4da8708cdc5713f526, ASSIGN 2024-11-24T22:14:39,054 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae, ASSIGN 2024-11-24T22:14:39,054 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=84dd509902edc43083bc78fe4b70922c, ASSIGN 2024-11-24T22:14:39,054 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e91326d9778777879bc895ecb241af8, ASSIGN 2024-11-24T22:14:39,054 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 516ead6fe1773c4da8708cdc5713f526, server=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:39,054 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure dc147c653b22684b8e41092cd6a0c8eb, server=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:39,054 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=73, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:39,055 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9b2e9ce57ffd0cd61c72602e5f6d9cae, server=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:39,055 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e91326d9778777879bc895ecb241af8, server=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:39,055 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure f72c935213180e3ff989af795022b7e4, server=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:39,055 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:39,055 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:39,055 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:39,055 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:39,056 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:39,056 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:39,056 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:39,056 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:39,056 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN 2024-11-24T22:14:39,056 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:39,056 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN 2024-11-24T22:14:39,056 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:39,056 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN 2024-11-24T22:14:39,056 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f72c935213180e3ff989af795022b7e4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:39,056 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f72c935213180e3ff989af795022b7e4, ASSIGN 2024-11-24T22:14:39,056 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=516ead6fe1773c4da8708cdc5713f526, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:39,056 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=516ead6fe1773c4da8708cdc5713f526, ASSIGN 2024-11-24T22:14:39,056 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:39,056 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae, ASSIGN 2024-11-24T22:14:39,056 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=84dd509902edc43083bc78fe4b70922c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:39,056 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=84dd509902edc43083bc78fe4b70922c, ASSIGN 2024-11-24T22:14:39,057 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e91326d9778777879bc895ecb241af8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:39,057 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e91326d9778777879bc895ecb241af8, ASSIGN 2024-11-24T22:14:39,057 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 516ead6fe1773c4da8708cdc5713f526, server=bd6592c37bb2,38125,1732486469405 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:39,057 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure dc147c653b22684b8e41092cd6a0c8eb, server=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:39,057 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=73, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:39,057 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9b2e9ce57ffd0cd61c72602e5f6d9cae, server=bd6592c37bb2,38125,1732486469405 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:39,057 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e91326d9778777879bc895ecb241af8, server=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:39,057 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure f72c935213180e3ff989af795022b7e4, server=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:39,057 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 18 msec 2024-11-24T22:14:39,073 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=65 2024-11-24T22:14:39,073 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=bd6592c37bb2,42639,1732486469476, table=hbase:meta, region=1588230740 2024-11-24T22:14:39,074 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bd6592c37bb2,42639,1732486469476, state=OPEN 2024-11-24T22:14:39,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36279-0x1014c0b79ef0000, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:39,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42639-0x1014c0b79ef0003, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:39,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38125-0x1014c0b79ef0002, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:39,077 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:39,077 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:39,077 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:39,078 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN to state=OFFLINE, location=null, table=test, region=dc147c653b22684b8e41092cd6a0c8eb to restore RIT 2024-11-24T22:14:39,078 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN to state=OFFLINE, location=null, table=test, region=fc9f96fd4c44240a85e6b2c001e46153 to restore RIT 2024-11-24T22:14:39,078 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN to state=OFFLINE, location=null, table=test, region=698dd9d64bdaf6a81e797d2590ff86f9 to restore RIT 2024-11-24T22:14:39,078 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f72c935213180e3ff989af795022b7e4, ASSIGN to state=OFFLINE, location=null, table=test, region=f72c935213180e3ff989af795022b7e4 to restore RIT 2024-11-24T22:14:39,078 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=516ead6fe1773c4da8708cdc5713f526, ASSIGN to state=OFFLINE, location=null, table=test, region=516ead6fe1773c4da8708cdc5713f526 to restore RIT 2024-11-24T22:14:39,079 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae, ASSIGN to state=OFFLINE, location=null, table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae to restore RIT 2024-11-24T22:14:39,079 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=84dd509902edc43083bc78fe4b70922c, ASSIGN to state=OFFLINE, location=null, table=test, region=84dd509902edc43083bc78fe4b70922c to restore RIT 2024-11-24T22:14:39,079 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e91326d9778777879bc895ecb241af8, ASSIGN to state=OFFLINE, location=null, table=test, region=4e91326d9778777879bc895ecb241af8 to restore RIT 2024-11-24T22:14:39,106 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732486509106 2024-11-24T22:14:39,106 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T22:14:39,108 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:39,111 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T22:14:39,113 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T22:14:39,113 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=79, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e91326d9778777879bc895ecb241af8, server=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:39,113 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T22:14:39,113 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=79, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e91326d9778777879bc895ecb241af8, server=bd6592c37bb2,42639,1732486469476 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:39,117 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e91326d9778777879bc895ecb241af8, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,117 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4279880609a62ccb9e9fb29e79bdf7e7, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,117 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c68a44a11eeaccf8a3c2f49ba1446c1e, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,118 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=698dd9d64bdaf6a81e797d2590ff86f9, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,118 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46b79f5a3db63445091682b5e72f635a, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,118 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=736f44b8f348e2a5a96dfd3abc5e62b2, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,118 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f72c935213180e3ff989af795022b7e4, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,119 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=72a41015f7b88194cc3a08feb66aa4f9, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,119 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94f2fa48145fe3bdeffbfded6a4edd73, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,119 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b2e9ce57ffd0cd61c72602e5f6d9cae, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,119 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b86957c7b81466a818bbff45fac751f, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,120 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=967acf99df098c28d8f837a9f0746a90, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,120 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc147c653b22684b8e41092cd6a0c8eb, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,120 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5160ff1dd0e56b08fbca2ffe581141d4, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,120 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce1f0bcb3026cb1cbbd9e63c21b83f99, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,120 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cca6a66154e5c299fad3a247afbfb5fd, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=5 2024-11-24T22:14:39,121 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83e58b0e0989e11c83283bf7bd41306c, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,121 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c802d82f160c5269119d007388afb85e, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,121 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=84dd509902edc43083bc78fe4b70922c, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,121 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acdab7e426c93b10b34b4d331b21cef0, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,122 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79c2dd6f654fabefe0fc06b8199e8562, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,122 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc9f96fd4c44240a85e6b2c001e46153, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,123 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50d9b38af2d6413e500860133d150aa8, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,123 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=516ead6fe1773c4da8708cdc5713f526, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,123 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79d315a497ec3ad891988cb800bae337, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,123 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed5b70c013e388180062b1513b980a07, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,123 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T22:14:39,123 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732486539123 2024-11-24T22:14:39,124 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732486599124 2024-11-24T22:14:39,124 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 17 msec 2024-11-24T22:14:39,124 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T22:14:39,124 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T22:14:39,124 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T22:14:39,124 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T22:14:39,126 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T22:14:39,126 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T22:14:39,126 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T22:14:39,126 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T22:14:39,128 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-24T22:14:39,141 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T22:14:39,141 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T22:14:39,141 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T22:14:39,141 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:39,141 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T22:14:39,141 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:39,141 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:39,141 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:39,141 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:39,141 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T22:14:39,141 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:39,141 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T22:14:39,141 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:39,141 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T22:14:39,141 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:39,141 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T22:14:39,141 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T22:14:39,141 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T22:14:39,141 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:39,141 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T22:14:39,142 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T22:14:39,143 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T22:14:39,143 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=cca6a66154e5c299fad3a247afbfb5fd, REOPEN/MOVE 2024-11-24T22:14:39,163 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true 2024-11-24T22:14:39,164 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:39,164 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:39,164 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T22:14:39,164 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:39,165 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:39,165 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:39,165 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN 2024-11-24T22:14:39,165 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN 2024-11-24T22:14:39,165 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN 2024-11-24T22:14:39,165 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f72c935213180e3ff989af795022b7e4, ASSIGN 2024-11-24T22:14:39,165 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=516ead6fe1773c4da8708cdc5713f526, ASSIGN 2024-11-24T22:14:39,165 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae, ASSIGN 2024-11-24T22:14:39,165 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=84dd509902edc43083bc78fe4b70922c, ASSIGN 2024-11-24T22:14:39,165 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e91326d9778777879bc895ecb241af8, ASSIGN 2024-11-24T22:14:39,165 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 516ead6fe1773c4da8708cdc5713f526, server=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:39,165 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure dc147c653b22684b8e41092cd6a0c8eb, server=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:39,165 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=73, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:39,165 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9b2e9ce57ffd0cd61c72602e5f6d9cae, server=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:39,165 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e91326d9778777879bc895ecb241af8, server=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:39,165 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:39,165 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:39,165 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:39,166 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:39,166 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:39,166 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:39,166 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:39,166 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:39,166 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN 2024-11-24T22:14:39,166 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:39,166 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN 2024-11-24T22:14:39,166 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:39,166 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN 2024-11-24T22:14:39,166 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f72c935213180e3ff989af795022b7e4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:39,166 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f72c935213180e3ff989af795022b7e4, ASSIGN 2024-11-24T22:14:39,166 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=516ead6fe1773c4da8708cdc5713f526, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:39,166 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=516ead6fe1773c4da8708cdc5713f526, ASSIGN 2024-11-24T22:14:39,166 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:39,166 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae, ASSIGN 2024-11-24T22:14:39,166 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=84dd509902edc43083bc78fe4b70922c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:39,166 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=84dd509902edc43083bc78fe4b70922c, ASSIGN 2024-11-24T22:14:39,166 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e91326d9778777879bc895ecb241af8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:39,166 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e91326d9778777879bc895ecb241af8, ASSIGN 2024-11-24T22:14:39,166 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 516ead6fe1773c4da8708cdc5713f526, server=bd6592c37bb2,38125,1732486469405 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:39,166 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure dc147c653b22684b8e41092cd6a0c8eb, server=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:39,166 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=73, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:39,166 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9b2e9ce57ffd0cd61c72602e5f6d9cae, server=bd6592c37bb2,38125,1732486469405 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:39,167 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e91326d9778777879bc895ecb241af8, server=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:39,167 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 38 msec 2024-11-24T22:14:39,175 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=65 2024-11-24T22:14:39,175 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=bd6592c37bb2,42639,1732486469476, table=hbase:meta, region=1588230740 2024-11-24T22:14:39,175 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bd6592c37bb2,42639,1732486469476, state=OPEN 2024-11-24T22:14:39,177 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42639-0x1014c0b79ef0003, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:39,177 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36279-0x1014c0b79ef0000, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:39,177 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38125-0x1014c0b79ef0002, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:39,178 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:39,178 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:39,178 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:39,179 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN to state=OFFLINE, location=null, table=test, region=dc147c653b22684b8e41092cd6a0c8eb to restore RIT 2024-11-24T22:14:39,179 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN to state=OFFLINE, location=null, table=test, region=fc9f96fd4c44240a85e6b2c001e46153 to restore RIT 2024-11-24T22:14:39,179 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN to state=OFFLINE, location=null, table=test, region=698dd9d64bdaf6a81e797d2590ff86f9 to restore RIT 2024-11-24T22:14:39,179 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f72c935213180e3ff989af795022b7e4, ASSIGN to state=OFFLINE, location=null, table=test, region=f72c935213180e3ff989af795022b7e4 to restore RIT 2024-11-24T22:14:39,179 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=516ead6fe1773c4da8708cdc5713f526, ASSIGN to state=OFFLINE, location=null, table=test, region=516ead6fe1773c4da8708cdc5713f526 to restore RIT 2024-11-24T22:14:39,179 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae, ASSIGN to state=OFFLINE, location=null, table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae to restore RIT 2024-11-24T22:14:39,179 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=84dd509902edc43083bc78fe4b70922c, ASSIGN to state=OFFLINE, location=null, table=test, region=84dd509902edc43083bc78fe4b70922c to restore RIT 2024-11-24T22:14:39,179 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e91326d9778777879bc895ecb241af8, ASSIGN to state=OFFLINE, location=null, table=test, region=4e91326d9778777879bc895ecb241af8 to restore RIT 2024-11-24T22:14:39,205 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732486509205 2024-11-24T22:14:39,205 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T22:14:39,216 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e91326d9778777879bc895ecb241af8, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,216 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4279880609a62ccb9e9fb29e79bdf7e7, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,217 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c68a44a11eeaccf8a3c2f49ba1446c1e, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,217 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=698dd9d64bdaf6a81e797d2590ff86f9, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,217 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46b79f5a3db63445091682b5e72f635a, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,217 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=736f44b8f348e2a5a96dfd3abc5e62b2, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,218 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f72c935213180e3ff989af795022b7e4, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,218 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=72a41015f7b88194cc3a08feb66aa4f9, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,218 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94f2fa48145fe3bdeffbfded6a4edd73, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,218 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b2e9ce57ffd0cd61c72602e5f6d9cae, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,218 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b86957c7b81466a818bbff45fac751f, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,219 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=967acf99df098c28d8f837a9f0746a90, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,219 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc147c653b22684b8e41092cd6a0c8eb, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,219 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5160ff1dd0e56b08fbca2ffe581141d4, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,219 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce1f0bcb3026cb1cbbd9e63c21b83f99, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cca6a66154e5c299fad3a247afbfb5fd, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=5 2024-11-24T22:14:39,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83e58b0e0989e11c83283bf7bd41306c, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c802d82f160c5269119d007388afb85e, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=84dd509902edc43083bc78fe4b70922c, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acdab7e426c93b10b34b4d331b21cef0, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79c2dd6f654fabefe0fc06b8199e8562, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc9f96fd4c44240a85e6b2c001e46153, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50d9b38af2d6413e500860133d150aa8, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=516ead6fe1773c4da8708cdc5713f526, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79d315a497ec3ad891988cb800bae337, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed5b70c013e388180062b1513b980a07, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,221 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T22:14:39,221 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732486539221 2024-11-24T22:14:39,221 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732486599221 2024-11-24T22:14:39,205 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:39,221 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 16 msec 2024-11-24T22:14:39,222 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T22:14:39,224 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T22:14:39,224 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=78, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9b2e9ce57ffd0cd61c72602e5f6d9cae, server=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:39,224 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T22:14:39,224 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=78, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9b2e9ce57ffd0cd61c72602e5f6d9cae, server=bd6592c37bb2,38125,1732486469405 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:39,472 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T22:14:39,472 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T22:14:39,472 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T22:14:39,472 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T22:14:39,473 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T22:14:39,473 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T22:14:39,473 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T22:14:39,473 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T22:14:39,474 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-24T22:14:39,479 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T22:14:39,479 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T22:14:39,479 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T22:14:39,479 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:39,479 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T22:14:39,479 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:39,479 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:39,479 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:39,479 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:39,479 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T22:14:39,479 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:39,479 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T22:14:39,479 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:39,479 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T22:14:39,479 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:39,479 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T22:14:39,479 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T22:14:39,479 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:39,479 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T22:14:39,480 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T22:14:39,481 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T22:14:39,481 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=cca6a66154e5c299fad3a247afbfb5fd, REOPEN/MOVE 2024-11-24T22:14:39,483 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true 2024-11-24T22:14:39,483 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:39,483 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:39,483 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T22:14:39,483 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:39,483 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:39,483 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:39,483 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN 2024-11-24T22:14:39,483 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN 2024-11-24T22:14:39,483 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN 2024-11-24T22:14:39,483 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f72c935213180e3ff989af795022b7e4, ASSIGN 2024-11-24T22:14:39,483 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=516ead6fe1773c4da8708cdc5713f526, ASSIGN 2024-11-24T22:14:39,483 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae, ASSIGN 2024-11-24T22:14:39,483 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=84dd509902edc43083bc78fe4b70922c, ASSIGN 2024-11-24T22:14:39,483 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e91326d9778777879bc895ecb241af8, ASSIGN 2024-11-24T22:14:39,483 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 516ead6fe1773c4da8708cdc5713f526, server=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:39,483 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure dc147c653b22684b8e41092cd6a0c8eb, server=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:39,483 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=73, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:39,483 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9b2e9ce57ffd0cd61c72602e5f6d9cae, server=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:39,483 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:39,483 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:39,483 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:39,483 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:39,483 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:39,484 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:39,484 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:39,484 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:39,484 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN 2024-11-24T22:14:39,484 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:39,484 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN 2024-11-24T22:14:39,484 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:39,484 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN 2024-11-24T22:14:39,484 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f72c935213180e3ff989af795022b7e4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:39,484 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f72c935213180e3ff989af795022b7e4, ASSIGN 2024-11-24T22:14:39,484 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=516ead6fe1773c4da8708cdc5713f526, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:39,484 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=516ead6fe1773c4da8708cdc5713f526, ASSIGN 2024-11-24T22:14:39,484 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:39,484 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae, ASSIGN 2024-11-24T22:14:39,484 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=84dd509902edc43083bc78fe4b70922c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:39,484 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=84dd509902edc43083bc78fe4b70922c, ASSIGN 2024-11-24T22:14:39,484 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e91326d9778777879bc895ecb241af8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:39,484 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e91326d9778777879bc895ecb241af8, ASSIGN 2024-11-24T22:14:39,484 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 516ead6fe1773c4da8708cdc5713f526, server=bd6592c37bb2,38125,1732486469405 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:39,484 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure dc147c653b22684b8e41092cd6a0c8eb, server=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:39,484 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=73, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:39,484 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9b2e9ce57ffd0cd61c72602e5f6d9cae, server=bd6592c37bb2,38125,1732486469405 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:39,484 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-24T22:14:39,485 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=65 2024-11-24T22:14:39,485 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=bd6592c37bb2,42639,1732486469476, table=hbase:meta, region=1588230740 2024-11-24T22:14:39,485 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bd6592c37bb2,42639,1732486469476, state=OPEN 2024-11-24T22:14:39,487 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38125-0x1014c0b79ef0002, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:39,487 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42639-0x1014c0b79ef0003, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:39,487 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36279-0x1014c0b79ef0000, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:39,487 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:39,487 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:39,487 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:39,489 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN to state=OFFLINE, location=null, table=test, region=dc147c653b22684b8e41092cd6a0c8eb to restore RIT 2024-11-24T22:14:39,489 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN to state=OFFLINE, location=null, table=test, region=fc9f96fd4c44240a85e6b2c001e46153 to restore RIT 2024-11-24T22:14:39,489 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN to state=OFFLINE, location=null, table=test, region=698dd9d64bdaf6a81e797d2590ff86f9 to restore RIT 2024-11-24T22:14:39,489 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f72c935213180e3ff989af795022b7e4, ASSIGN to state=OFFLINE, location=null, table=test, region=f72c935213180e3ff989af795022b7e4 to restore RIT 2024-11-24T22:14:39,489 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=516ead6fe1773c4da8708cdc5713f526, ASSIGN to state=OFFLINE, location=null, table=test, region=516ead6fe1773c4da8708cdc5713f526 to restore RIT 2024-11-24T22:14:39,489 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae, ASSIGN to state=OFFLINE, location=null, table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae to restore RIT 2024-11-24T22:14:39,489 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=84dd509902edc43083bc78fe4b70922c, ASSIGN to state=OFFLINE, location=null, table=test, region=84dd509902edc43083bc78fe4b70922c to restore RIT 2024-11-24T22:14:39,489 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e91326d9778777879bc895ecb241af8, ASSIGN to state=OFFLINE, location=null, table=test, region=4e91326d9778777879bc895ecb241af8 to restore RIT 2024-11-24T22:14:39,496 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732486509496 2024-11-24T22:14:39,496 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T22:14:39,498 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:39,500 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T22:14:39,501 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T22:14:39,501 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=77, ppid=73, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:39,501 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T22:14:39,501 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=77, ppid=73, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:39,504 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e91326d9778777879bc895ecb241af8, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,504 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4279880609a62ccb9e9fb29e79bdf7e7, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,504 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c68a44a11eeaccf8a3c2f49ba1446c1e, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,505 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=698dd9d64bdaf6a81e797d2590ff86f9, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,505 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46b79f5a3db63445091682b5e72f635a, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,505 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=736f44b8f348e2a5a96dfd3abc5e62b2, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,505 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f72c935213180e3ff989af795022b7e4, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,505 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=72a41015f7b88194cc3a08feb66aa4f9, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,506 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94f2fa48145fe3bdeffbfded6a4edd73, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,506 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b2e9ce57ffd0cd61c72602e5f6d9cae, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,506 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b86957c7b81466a818bbff45fac751f, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,507 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=967acf99df098c28d8f837a9f0746a90, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,507 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc147c653b22684b8e41092cd6a0c8eb, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,507 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5160ff1dd0e56b08fbca2ffe581141d4, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,507 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce1f0bcb3026cb1cbbd9e63c21b83f99, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,507 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cca6a66154e5c299fad3a247afbfb5fd, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=5 2024-11-24T22:14:39,508 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83e58b0e0989e11c83283bf7bd41306c, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,508 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c802d82f160c5269119d007388afb85e, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,508 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=84dd509902edc43083bc78fe4b70922c, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,508 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acdab7e426c93b10b34b4d331b21cef0, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,509 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79c2dd6f654fabefe0fc06b8199e8562, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,509 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc9f96fd4c44240a85e6b2c001e46153, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,509 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50d9b38af2d6413e500860133d150aa8, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,509 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=516ead6fe1773c4da8708cdc5713f526, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,509 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79d315a497ec3ad891988cb800bae337, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,510 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed5b70c013e388180062b1513b980a07, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,510 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T22:14:39,510 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732486539510 2024-11-24T22:14:39,510 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732486599510 2024-11-24T22:14:39,510 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 13 msec 2024-11-24T22:14:39,510 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T22:14:39,510 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T22:14:39,510 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T22:14:39,510 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T22:14:39,510 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T22:14:39,510 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T22:14:39,510 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T22:14:39,511 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T22:14:39,512 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-24T22:14:39,514 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T22:14:39,514 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T22:14:39,514 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T22:14:39,515 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:39,515 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T22:14:39,515 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:39,515 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:39,515 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:39,515 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:39,515 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T22:14:39,515 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:39,515 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:39,515 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T22:14:39,515 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:39,515 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T22:14:39,515 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T22:14:39,515 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:39,515 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T22:14:39,515 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T22:14:39,519 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T22:14:39,519 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=cca6a66154e5c299fad3a247afbfb5fd, REOPEN/MOVE 2024-11-24T22:14:39,522 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true 2024-11-24T22:14:39,522 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:39,522 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:39,522 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T22:14:39,522 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:39,522 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:39,522 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:39,522 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN 2024-11-24T22:14:39,522 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN 2024-11-24T22:14:39,522 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN 2024-11-24T22:14:39,523 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f72c935213180e3ff989af795022b7e4, ASSIGN 2024-11-24T22:14:39,523 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=516ead6fe1773c4da8708cdc5713f526, ASSIGN 2024-11-24T22:14:39,523 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae, ASSIGN 2024-11-24T22:14:39,523 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=84dd509902edc43083bc78fe4b70922c, ASSIGN 2024-11-24T22:14:39,523 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e91326d9778777879bc895ecb241af8, ASSIGN 2024-11-24T22:14:39,523 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 516ead6fe1773c4da8708cdc5713f526, server=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:39,523 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure dc147c653b22684b8e41092cd6a0c8eb, server=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:39,524 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=73, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:39,524 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:39,524 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:39,524 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:39,524 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:39,525 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:39,525 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:39,526 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:39,526 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:39,526 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN 2024-11-24T22:14:39,526 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:39,526 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN 2024-11-24T22:14:39,526 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:39,526 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN 2024-11-24T22:14:39,526 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f72c935213180e3ff989af795022b7e4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:39,526 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f72c935213180e3ff989af795022b7e4, ASSIGN 2024-11-24T22:14:39,526 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=516ead6fe1773c4da8708cdc5713f526, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:39,526 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=516ead6fe1773c4da8708cdc5713f526, ASSIGN 2024-11-24T22:14:39,526 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:39,527 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36279 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:39,527 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae, ASSIGN 2024-11-24T22:14:39,527 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=84dd509902edc43083bc78fe4b70922c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:39,527 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=84dd509902edc43083bc78fe4b70922c, ASSIGN 2024-11-24T22:14:39,527 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e91326d9778777879bc895ecb241af8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:39,527 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e91326d9778777879bc895ecb241af8, ASSIGN 2024-11-24T22:14:39,527 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 516ead6fe1773c4da8708cdc5713f526, server=bd6592c37bb2,38125,1732486469405 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:39,527 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure dc147c653b22684b8e41092cd6a0c8eb, server=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:39,527 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36279 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:39,528 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=73, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:39,529 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 15 msec 2024-11-24T22:14:39,537 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=65 2024-11-24T22:14:39,537 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=bd6592c37bb2,42639,1732486469476, table=hbase:meta, region=1588230740 2024-11-24T22:14:39,538 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bd6592c37bb2,42639,1732486469476, state=OPEN 2024-11-24T22:14:39,540 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36279-0x1014c0b79ef0000, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:39,540 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42639-0x1014c0b79ef0003, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:39,540 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:39,540 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:39,540 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38125-0x1014c0b79ef0002, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:39,541 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:39,557 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN to state=OFFLINE, location=null, table=test, region=dc147c653b22684b8e41092cd6a0c8eb to restore RIT 2024-11-24T22:14:39,558 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN to state=OFFLINE, location=null, table=test, region=fc9f96fd4c44240a85e6b2c001e46153 to restore RIT 2024-11-24T22:14:39,558 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN to state=OFFLINE, location=null, table=test, region=698dd9d64bdaf6a81e797d2590ff86f9 to restore RIT 2024-11-24T22:14:39,559 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f72c935213180e3ff989af795022b7e4, ASSIGN to state=OFFLINE, location=null, table=test, region=f72c935213180e3ff989af795022b7e4 to restore RIT 2024-11-24T22:14:39,559 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=516ead6fe1773c4da8708cdc5713f526, ASSIGN to state=OFFLINE, location=null, table=test, region=516ead6fe1773c4da8708cdc5713f526 to restore RIT 2024-11-24T22:14:39,559 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae, ASSIGN to state=OFFLINE, location=null, table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae to restore RIT 2024-11-24T22:14:39,559 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=84dd509902edc43083bc78fe4b70922c, ASSIGN to state=OFFLINE, location=null, table=test, region=84dd509902edc43083bc78fe4b70922c to restore RIT 2024-11-24T22:14:39,559 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e91326d9778777879bc895ecb241af8, ASSIGN to state=OFFLINE, location=null, table=test, region=4e91326d9778777879bc895ecb241af8 to restore RIT 2024-11-24T22:14:39,559 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732486509559 2024-11-24T22:14:39,559 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T22:14:39,567 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e91326d9778777879bc895ecb241af8, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,567 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4279880609a62ccb9e9fb29e79bdf7e7, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,568 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c68a44a11eeaccf8a3c2f49ba1446c1e, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,568 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=698dd9d64bdaf6a81e797d2590ff86f9, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,568 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46b79f5a3db63445091682b5e72f635a, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,568 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=736f44b8f348e2a5a96dfd3abc5e62b2, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,569 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f72c935213180e3ff989af795022b7e4, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,569 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=72a41015f7b88194cc3a08feb66aa4f9, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,569 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94f2fa48145fe3bdeffbfded6a4edd73, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,569 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b2e9ce57ffd0cd61c72602e5f6d9cae, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,570 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b86957c7b81466a818bbff45fac751f, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,570 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=967acf99df098c28d8f837a9f0746a90, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,570 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc147c653b22684b8e41092cd6a0c8eb, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,570 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5160ff1dd0e56b08fbca2ffe581141d4, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,570 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce1f0bcb3026cb1cbbd9e63c21b83f99, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,570 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cca6a66154e5c299fad3a247afbfb5fd, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=5 2024-11-24T22:14:39,571 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83e58b0e0989e11c83283bf7bd41306c, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,571 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c802d82f160c5269119d007388afb85e, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,571 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=84dd509902edc43083bc78fe4b70922c, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,571 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acdab7e426c93b10b34b4d331b21cef0, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,571 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79c2dd6f654fabefe0fc06b8199e8562, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,571 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc9f96fd4c44240a85e6b2c001e46153, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,572 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50d9b38af2d6413e500860133d150aa8, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,572 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=516ead6fe1773c4da8708cdc5713f526, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,572 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79d315a497ec3ad891988cb800bae337, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,572 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed5b70c013e388180062b1513b980a07, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,572 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T22:14:39,572 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732486539572 2024-11-24T22:14:39,572 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732486599572 2024-11-24T22:14:39,572 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 12 msec 2024-11-24T22:14:39,561 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:39,580 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T22:14:39,586 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T22:14:39,586 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure dc147c653b22684b8e41092cd6a0c8eb, server=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:39,587 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T22:14:39,587 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure dc147c653b22684b8e41092cd6a0c8eb, server=bd6592c37bb2,42639,1732486469476 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:39,823 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T22:14:39,823 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T22:14:39,823 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T22:14:39,823 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T22:14:39,823 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T22:14:39,824 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T22:14:39,824 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T22:14:39,824 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T22:14:39,825 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-24T22:14:39,827 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T22:14:39,827 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T22:14:39,827 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T22:14:39,827 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:39,827 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T22:14:39,827 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:39,827 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:39,827 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:39,827 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:39,827 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T22:14:39,827 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:39,827 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:39,827 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:39,828 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T22:14:39,828 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T22:14:39,828 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:39,828 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T22:14:39,828 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T22:14:39,829 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T22:14:39,829 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=cca6a66154e5c299fad3a247afbfb5fd, REOPEN/MOVE 2024-11-24T22:14:39,831 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true 2024-11-24T22:14:39,831 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:39,831 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:39,832 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T22:14:39,832 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:39,832 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:39,832 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:39,832 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN 2024-11-24T22:14:39,832 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN 2024-11-24T22:14:39,832 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN 2024-11-24T22:14:39,832 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f72c935213180e3ff989af795022b7e4, ASSIGN 2024-11-24T22:14:39,832 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=516ead6fe1773c4da8708cdc5713f526, ASSIGN 2024-11-24T22:14:39,832 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae, ASSIGN 2024-11-24T22:14:39,832 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=84dd509902edc43083bc78fe4b70922c, ASSIGN 2024-11-24T22:14:39,832 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e91326d9778777879bc895ecb241af8, ASSIGN 2024-11-24T22:14:39,832 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 516ead6fe1773c4da8708cdc5713f526, server=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:39,832 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure dc147c653b22684b8e41092cd6a0c8eb, server=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:39,832 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:39,832 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:39,832 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:39,832 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:39,832 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:39,832 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:39,832 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:39,832 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:39,832 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN 2024-11-24T22:14:39,832 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:39,832 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN 2024-11-24T22:14:39,832 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:39,833 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN 2024-11-24T22:14:39,833 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f72c935213180e3ff989af795022b7e4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:39,833 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f72c935213180e3ff989af795022b7e4, ASSIGN 2024-11-24T22:14:39,833 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=516ead6fe1773c4da8708cdc5713f526, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:39,833 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=516ead6fe1773c4da8708cdc5713f526, ASSIGN 2024-11-24T22:14:39,833 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:39,833 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae, ASSIGN 2024-11-24T22:14:39,833 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=84dd509902edc43083bc78fe4b70922c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:39,833 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=84dd509902edc43083bc78fe4b70922c, ASSIGN 2024-11-24T22:14:39,833 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e91326d9778777879bc895ecb241af8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:39,833 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e91326d9778777879bc895ecb241af8, ASSIGN 2024-11-24T22:14:39,833 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 516ead6fe1773c4da8708cdc5713f526, server=bd6592c37bb2,38125,1732486469405 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:39,833 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure dc147c653b22684b8e41092cd6a0c8eb, server=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:39,833 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-24T22:14:39,834 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=65 2024-11-24T22:14:39,834 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=bd6592c37bb2,42639,1732486469476, table=hbase:meta, region=1588230740 2024-11-24T22:14:39,834 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bd6592c37bb2,42639,1732486469476, state=OPEN 2024-11-24T22:14:39,834 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36279 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:39,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36279-0x1014c0b79ef0000, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:39,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42639-0x1014c0b79ef0003, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:39,836 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:39,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38125-0x1014c0b79ef0002, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:39,837 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:39,837 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:39,839 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "bd6592c37bb2" port: 38125 start_code: 1732486469405 } transition { transition_code: OPENED region_info { region_id: 1732486472486 table_name { namespace: "default" qualifier: "test" } start_key: "ddd" end_key: "eee" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 82 initiating_master_active_time: 1732486469596 }; retry (#1) after 2016ms delay (Master is coming online...). org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-24T22:14:39,840 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36279 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:39,840 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36279 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:39,841 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36279 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:39,842 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "bd6592c37bb2" port: 42639 start_code: 1732486469476 } transition { transition_code: OPENED region_info { region_id: 1732486472486 table_name { namespace: "default" qualifier: "test" } start_key: "" end_key: "bbb" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 79 initiating_master_active_time: 1732486469596 }; retry (#1) after 2006ms delay (Master is coming online...). org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-24T22:14:39,842 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN to state=OFFLINE, location=null, table=test, region=dc147c653b22684b8e41092cd6a0c8eb to restore RIT 2024-11-24T22:14:39,842 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN to state=OFFLINE, location=null, table=test, region=fc9f96fd4c44240a85e6b2c001e46153 to restore RIT 2024-11-24T22:14:39,843 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN to state=OFFLINE, location=null, table=test, region=698dd9d64bdaf6a81e797d2590ff86f9 to restore RIT 2024-11-24T22:14:39,843 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f72c935213180e3ff989af795022b7e4, ASSIGN to state=OFFLINE, location=null, table=test, region=f72c935213180e3ff989af795022b7e4 to restore RIT 2024-11-24T22:14:39,843 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=516ead6fe1773c4da8708cdc5713f526, ASSIGN to state=OFFLINE, location=null, table=test, region=516ead6fe1773c4da8708cdc5713f526 to restore RIT 2024-11-24T22:14:39,843 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae, ASSIGN to state=OFFLINE, location=null, table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae to restore RIT 2024-11-24T22:14:39,843 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=84dd509902edc43083bc78fe4b70922c, ASSIGN to state=OFFLINE, location=null, table=test, region=84dd509902edc43083bc78fe4b70922c to restore RIT 2024-11-24T22:14:39,843 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e91326d9778777879bc895ecb241af8, ASSIGN to state=OFFLINE, location=null, table=test, region=4e91326d9778777879bc895ecb241af8 to restore RIT 2024-11-24T22:14:39,850 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732486509849 2024-11-24T22:14:39,850 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T22:14:39,851 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:39,853 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T22:14:39,856 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T22:14:39,856 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 516ead6fe1773c4da8708cdc5713f526, server=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:39,856 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T22:14:39,856 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 516ead6fe1773c4da8708cdc5713f526, server=bd6592c37bb2,38125,1732486469405 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:39,857 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e91326d9778777879bc895ecb241af8, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,857 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4279880609a62ccb9e9fb29e79bdf7e7, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,858 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c68a44a11eeaccf8a3c2f49ba1446c1e, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,858 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=698dd9d64bdaf6a81e797d2590ff86f9, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,858 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46b79f5a3db63445091682b5e72f635a, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,858 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=736f44b8f348e2a5a96dfd3abc5e62b2, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,858 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f72c935213180e3ff989af795022b7e4, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,859 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=72a41015f7b88194cc3a08feb66aa4f9, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,859 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94f2fa48145fe3bdeffbfded6a4edd73, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,859 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b2e9ce57ffd0cd61c72602e5f6d9cae, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,859 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b86957c7b81466a818bbff45fac751f, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,859 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=967acf99df098c28d8f837a9f0746a90, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,859 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc147c653b22684b8e41092cd6a0c8eb, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,859 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5160ff1dd0e56b08fbca2ffe581141d4, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,860 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce1f0bcb3026cb1cbbd9e63c21b83f99, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,860 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cca6a66154e5c299fad3a247afbfb5fd, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=5 2024-11-24T22:14:39,860 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83e58b0e0989e11c83283bf7bd41306c, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,860 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c802d82f160c5269119d007388afb85e, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,860 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=84dd509902edc43083bc78fe4b70922c, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,861 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acdab7e426c93b10b34b4d331b21cef0, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,861 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79c2dd6f654fabefe0fc06b8199e8562, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,861 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc9f96fd4c44240a85e6b2c001e46153, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,861 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50d9b38af2d6413e500860133d150aa8, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,861 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=516ead6fe1773c4da8708cdc5713f526, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,862 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79d315a497ec3ad891988cb800bae337, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,862 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed5b70c013e388180062b1513b980a07, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,862 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T22:14:39,862 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732486539862 2024-11-24T22:14:39,862 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732486599862 2024-11-24T22:14:39,862 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 12 msec 2024-11-24T22:14:39,862 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T22:14:39,862 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T22:14:39,862 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T22:14:39,863 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T22:14:39,863 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T22:14:39,863 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T22:14:39,863 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T22:14:39,863 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T22:14:39,865 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-24T22:14:39,867 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T22:14:39,867 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T22:14:39,867 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T22:14:39,867 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:39,867 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:39,867 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:39,867 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:39,867 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:39,867 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T22:14:39,867 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:39,867 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:39,867 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:39,867 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T22:14:39,867 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T22:14:39,867 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:39,868 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T22:14:39,868 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T22:14:39,869 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T22:14:39,869 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=cca6a66154e5c299fad3a247afbfb5fd, REOPEN/MOVE 2024-11-24T22:14:39,873 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true 2024-11-24T22:14:39,873 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:39,873 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:39,873 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T22:14:39,873 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:39,873 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:39,873 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:39,873 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN 2024-11-24T22:14:39,873 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN 2024-11-24T22:14:39,873 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN 2024-11-24T22:14:39,873 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f72c935213180e3ff989af795022b7e4, ASSIGN 2024-11-24T22:14:39,873 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=516ead6fe1773c4da8708cdc5713f526, ASSIGN 2024-11-24T22:14:39,873 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae, ASSIGN 2024-11-24T22:14:39,873 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=84dd509902edc43083bc78fe4b70922c, ASSIGN 2024-11-24T22:14:39,873 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e91326d9778777879bc895ecb241af8, ASSIGN 2024-11-24T22:14:39,873 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 516ead6fe1773c4da8708cdc5713f526, server=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:39,874 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:39,874 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:39,874 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:39,874 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:39,874 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:39,874 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:39,874 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:39,874 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:39,874 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN 2024-11-24T22:14:39,874 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:39,874 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN 2024-11-24T22:14:39,874 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:39,874 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN 2024-11-24T22:14:39,874 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f72c935213180e3ff989af795022b7e4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:39,874 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f72c935213180e3ff989af795022b7e4, ASSIGN 2024-11-24T22:14:39,874 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=516ead6fe1773c4da8708cdc5713f526, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:39,874 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=516ead6fe1773c4da8708cdc5713f526, ASSIGN 2024-11-24T22:14:39,874 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:39,874 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae, ASSIGN 2024-11-24T22:14:39,874 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=84dd509902edc43083bc78fe4b70922c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:39,874 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=84dd509902edc43083bc78fe4b70922c, ASSIGN 2024-11-24T22:14:39,874 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e91326d9778777879bc895ecb241af8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:39,874 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e91326d9778777879bc895ecb241af8, ASSIGN 2024-11-24T22:14:39,874 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 516ead6fe1773c4da8708cdc5713f526, server=bd6592c37bb2,38125,1732486469405 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:39,874 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-24T22:14:39,877 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=65 2024-11-24T22:14:39,877 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=bd6592c37bb2,42639,1732486469476, table=hbase:meta, region=1588230740 2024-11-24T22:14:39,877 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bd6592c37bb2,42639,1732486469476, state=OPEN 2024-11-24T22:14:39,880 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36279-0x1014c0b79ef0000, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:39,880 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42639-0x1014c0b79ef0003, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:39,880 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38125-0x1014c0b79ef0002, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:39,880 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:39,880 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:39,880 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:39,881 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN to state=OFFLINE, location=null, table=test, region=dc147c653b22684b8e41092cd6a0c8eb to restore RIT 2024-11-24T22:14:39,881 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN to state=OFFLINE, location=null, table=test, region=fc9f96fd4c44240a85e6b2c001e46153 to restore RIT 2024-11-24T22:14:39,881 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN to state=OFFLINE, location=null, table=test, region=698dd9d64bdaf6a81e797d2590ff86f9 to restore RIT 2024-11-24T22:14:39,881 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f72c935213180e3ff989af795022b7e4, ASSIGN to state=OFFLINE, location=null, table=test, region=f72c935213180e3ff989af795022b7e4 to restore RIT 2024-11-24T22:14:39,882 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=516ead6fe1773c4da8708cdc5713f526, ASSIGN to state=OFFLINE, location=null, table=test, region=516ead6fe1773c4da8708cdc5713f526 to restore RIT 2024-11-24T22:14:39,882 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae, ASSIGN to state=OFFLINE, location=null, table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae to restore RIT 2024-11-24T22:14:39,882 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=84dd509902edc43083bc78fe4b70922c, ASSIGN to state=OFFLINE, location=null, table=test, region=84dd509902edc43083bc78fe4b70922c to restore RIT 2024-11-24T22:14:39,882 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e91326d9778777879bc895ecb241af8, ASSIGN to state=OFFLINE, location=null, table=test, region=4e91326d9778777879bc895ecb241af8 to restore RIT 2024-11-24T22:14:39,888 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732486509888 2024-11-24T22:14:39,888 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T22:14:39,889 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:39,893 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T22:14:39,895 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T22:14:39,895 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e91326d9778777879bc895ecb241af8, ASSIGN 2024-11-24T22:14:39,895 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T22:14:39,895 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e91326d9778777879bc895ecb241af8, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:39,896 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e91326d9778777879bc895ecb241af8, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,896 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4279880609a62ccb9e9fb29e79bdf7e7, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,896 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c68a44a11eeaccf8a3c2f49ba1446c1e, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,896 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=698dd9d64bdaf6a81e797d2590ff86f9, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,896 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46b79f5a3db63445091682b5e72f635a, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,896 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=736f44b8f348e2a5a96dfd3abc5e62b2, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,896 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f72c935213180e3ff989af795022b7e4, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,897 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=72a41015f7b88194cc3a08feb66aa4f9, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,897 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94f2fa48145fe3bdeffbfded6a4edd73, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,897 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b2e9ce57ffd0cd61c72602e5f6d9cae, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,897 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b86957c7b81466a818bbff45fac751f, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,897 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=967acf99df098c28d8f837a9f0746a90, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,897 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc147c653b22684b8e41092cd6a0c8eb, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,897 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5160ff1dd0e56b08fbca2ffe581141d4, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,897 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce1f0bcb3026cb1cbbd9e63c21b83f99, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,898 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cca6a66154e5c299fad3a247afbfb5fd, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=5 2024-11-24T22:14:39,898 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83e58b0e0989e11c83283bf7bd41306c, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,898 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c802d82f160c5269119d007388afb85e, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,898 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=84dd509902edc43083bc78fe4b70922c, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,898 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acdab7e426c93b10b34b4d331b21cef0, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,898 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79c2dd6f654fabefe0fc06b8199e8562, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,899 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc9f96fd4c44240a85e6b2c001e46153, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,899 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50d9b38af2d6413e500860133d150aa8, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,899 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=516ead6fe1773c4da8708cdc5713f526, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,899 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79d315a497ec3ad891988cb800bae337, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,899 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed5b70c013e388180062b1513b980a07, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,899 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T22:14:39,899 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732486539899 2024-11-24T22:14:39,899 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732486599899 2024-11-24T22:14:39,899 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 11 msec 2024-11-24T22:14:39,899 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T22:14:39,899 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T22:14:39,899 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T22:14:39,900 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T22:14:39,900 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T22:14:39,900 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T22:14:39,900 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T22:14:39,901 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T22:14:39,902 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-24T22:14:39,905 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T22:14:39,905 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T22:14:39,905 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T22:14:39,905 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:39,905 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:39,905 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:39,905 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:39,905 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:39,905 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:39,905 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:39,905 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:39,905 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T22:14:39,906 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T22:14:39,907 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:39,907 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T22:14:39,907 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T22:14:39,908 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T22:14:39,908 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=cca6a66154e5c299fad3a247afbfb5fd, REOPEN/MOVE 2024-11-24T22:14:39,910 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true 2024-11-24T22:14:39,910 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:39,910 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:39,910 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T22:14:39,910 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:39,910 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:39,910 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:39,910 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN 2024-11-24T22:14:39,910 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN 2024-11-24T22:14:39,910 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN 2024-11-24T22:14:39,910 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f72c935213180e3ff989af795022b7e4, ASSIGN 2024-11-24T22:14:39,910 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=516ead6fe1773c4da8708cdc5713f526, ASSIGN 2024-11-24T22:14:39,910 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae, ASSIGN 2024-11-24T22:14:39,910 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=84dd509902edc43083bc78fe4b70922c, ASSIGN 2024-11-24T22:14:39,910 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e91326d9778777879bc895ecb241af8, ASSIGN 2024-11-24T22:14:39,910 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:39,910 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:39,910 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:39,911 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:39,911 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:39,911 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:39,911 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:39,911 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:39,911 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN 2024-11-24T22:14:39,911 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:39,911 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN 2024-11-24T22:14:39,911 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:39,911 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN 2024-11-24T22:14:39,911 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f72c935213180e3ff989af795022b7e4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:39,911 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f72c935213180e3ff989af795022b7e4, ASSIGN 2024-11-24T22:14:39,911 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=516ead6fe1773c4da8708cdc5713f526, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:39,911 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=516ead6fe1773c4da8708cdc5713f526, ASSIGN 2024-11-24T22:14:39,911 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:39,911 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae, ASSIGN 2024-11-24T22:14:39,911 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=84dd509902edc43083bc78fe4b70922c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:39,911 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=84dd509902edc43083bc78fe4b70922c, ASSIGN 2024-11-24T22:14:39,911 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e91326d9778777879bc895ecb241af8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:39,911 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e91326d9778777879bc895ecb241af8, ASSIGN 2024-11-24T22:14:39,911 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-11-24T22:14:39,914 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=65 2024-11-24T22:14:39,914 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=bd6592c37bb2,42639,1732486469476, table=hbase:meta, region=1588230740 2024-11-24T22:14:39,915 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bd6592c37bb2,42639,1732486469476, state=OPEN 2024-11-24T22:14:39,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36279-0x1014c0b79ef0000, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:39,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42639-0x1014c0b79ef0003, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:39,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38125-0x1014c0b79ef0002, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:39,917 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:39,917 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:39,917 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:39,920 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN to state=OFFLINE, location=null, table=test, region=dc147c653b22684b8e41092cd6a0c8eb to restore RIT 2024-11-24T22:14:39,920 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN to state=OFFLINE, location=null, table=test, region=fc9f96fd4c44240a85e6b2c001e46153 to restore RIT 2024-11-24T22:14:39,920 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN to state=OFFLINE, location=null, table=test, region=698dd9d64bdaf6a81e797d2590ff86f9 to restore RIT 2024-11-24T22:14:39,920 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f72c935213180e3ff989af795022b7e4, ASSIGN to state=OFFLINE, location=null, table=test, region=f72c935213180e3ff989af795022b7e4 to restore RIT 2024-11-24T22:14:39,921 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=516ead6fe1773c4da8708cdc5713f526, ASSIGN to state=OFFLINE, location=null, table=test, region=516ead6fe1773c4da8708cdc5713f526 to restore RIT 2024-11-24T22:14:39,921 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae, ASSIGN to state=OFFLINE, location=null, table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae to restore RIT 2024-11-24T22:14:39,921 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=84dd509902edc43083bc78fe4b70922c, ASSIGN to state=OFFLINE, location=null, table=test, region=84dd509902edc43083bc78fe4b70922c to restore RIT 2024-11-24T22:14:39,921 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4e91326d9778777879bc895ecb241af8, ASSIGN to state=OFFLINE, location=null, table=test, region=4e91326d9778777879bc895ecb241af8 to restore RIT 2024-11-24T22:14:39,929 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732486509929 2024-11-24T22:14:39,929 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T22:14:39,931 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:39,935 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T22:14:39,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e91326d9778777879bc895ecb241af8, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4279880609a62ccb9e9fb29e79bdf7e7, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c68a44a11eeaccf8a3c2f49ba1446c1e, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=698dd9d64bdaf6a81e797d2590ff86f9, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46b79f5a3db63445091682b5e72f635a, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,938 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=736f44b8f348e2a5a96dfd3abc5e62b2, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,938 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f72c935213180e3ff989af795022b7e4, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,938 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=72a41015f7b88194cc3a08feb66aa4f9, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,938 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94f2fa48145fe3bdeffbfded6a4edd73, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,938 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b2e9ce57ffd0cd61c72602e5f6d9cae, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,938 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b86957c7b81466a818bbff45fac751f, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,939 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=967acf99df098c28d8f837a9f0746a90, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,939 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc147c653b22684b8e41092cd6a0c8eb, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,939 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5160ff1dd0e56b08fbca2ffe581141d4, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,939 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce1f0bcb3026cb1cbbd9e63c21b83f99, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,940 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cca6a66154e5c299fad3a247afbfb5fd, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=5 2024-11-24T22:14:39,940 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83e58b0e0989e11c83283bf7bd41306c, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,940 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c802d82f160c5269119d007388afb85e, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,940 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=84dd509902edc43083bc78fe4b70922c, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,940 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acdab7e426c93b10b34b4d331b21cef0, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,940 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79c2dd6f654fabefe0fc06b8199e8562, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,941 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc9f96fd4c44240a85e6b2c001e46153, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,941 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50d9b38af2d6413e500860133d150aa8, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,941 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=516ead6fe1773c4da8708cdc5713f526, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,941 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79d315a497ec3ad891988cb800bae337, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:39,941 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed5b70c013e388180062b1513b980a07, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:39,941 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T22:14:39,941 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732486539941 2024-11-24T22:14:39,941 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732486599941 2024-11-24T22:14:39,942 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 12 msec 2024-11-24T22:14:39,942 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T22:14:39,942 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=84dd509902edc43083bc78fe4b70922c, ASSIGN 2024-11-24T22:14:39,942 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T22:14:39,942 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=84dd509902edc43083bc78fe4b70922c, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:40,192 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T22:14:40,192 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T22:14:40,192 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T22:14:40,192 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T22:14:40,192 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T22:14:40,193 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T22:14:40,193 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T22:14:40,193 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T22:14:40,194 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-24T22:14:40,196 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T22:14:40,196 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T22:14:40,196 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T22:14:40,196 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:40,196 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:40,196 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:40,196 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:40,196 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:40,196 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:40,196 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:40,196 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T22:14:40,196 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T22:14:40,196 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:40,196 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T22:14:40,196 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T22:14:40,197 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T22:14:40,197 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=cca6a66154e5c299fad3a247afbfb5fd, REOPEN/MOVE 2024-11-24T22:14:40,199 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true 2024-11-24T22:14:40,199 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:40,199 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:40,199 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T22:14:40,199 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:40,199 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:40,199 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:40,199 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN 2024-11-24T22:14:40,199 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN 2024-11-24T22:14:40,200 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN 2024-11-24T22:14:40,200 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f72c935213180e3ff989af795022b7e4, ASSIGN 2024-11-24T22:14:40,200 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=516ead6fe1773c4da8708cdc5713f526, ASSIGN 2024-11-24T22:14:40,200 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae, ASSIGN 2024-11-24T22:14:40,200 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=84dd509902edc43083bc78fe4b70922c, ASSIGN 2024-11-24T22:14:40,200 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:40,200 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:40,200 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:40,200 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:40,200 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:40,200 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:40,200 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:40,200 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:40,200 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN 2024-11-24T22:14:40,200 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:40,200 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN 2024-11-24T22:14:40,200 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:40,200 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN 2024-11-24T22:14:40,200 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f72c935213180e3ff989af795022b7e4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:40,200 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f72c935213180e3ff989af795022b7e4, ASSIGN 2024-11-24T22:14:40,200 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=516ead6fe1773c4da8708cdc5713f526, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:40,200 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=516ead6fe1773c4da8708cdc5713f526, ASSIGN 2024-11-24T22:14:40,200 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:40,200 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae, ASSIGN 2024-11-24T22:14:40,200 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=84dd509902edc43083bc78fe4b70922c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:40,200 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=84dd509902edc43083bc78fe4b70922c, ASSIGN 2024-11-24T22:14:40,200 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-24T22:14:40,201 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=65 2024-11-24T22:14:40,201 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=bd6592c37bb2,42639,1732486469476, table=hbase:meta, region=1588230740 2024-11-24T22:14:40,202 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bd6592c37bb2,42639,1732486469476, state=OPEN 2024-11-24T22:14:40,204 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42639-0x1014c0b79ef0003, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:40,204 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38125-0x1014c0b79ef0002, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:40,204 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36279-0x1014c0b79ef0000, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:40,204 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:40,204 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:40,204 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:40,205 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN to state=OFFLINE, location=null, table=test, region=dc147c653b22684b8e41092cd6a0c8eb to restore RIT 2024-11-24T22:14:40,205 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN to state=OFFLINE, location=null, table=test, region=fc9f96fd4c44240a85e6b2c001e46153 to restore RIT 2024-11-24T22:14:40,205 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN to state=OFFLINE, location=null, table=test, region=698dd9d64bdaf6a81e797d2590ff86f9 to restore RIT 2024-11-24T22:14:40,205 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f72c935213180e3ff989af795022b7e4, ASSIGN to state=OFFLINE, location=null, table=test, region=f72c935213180e3ff989af795022b7e4 to restore RIT 2024-11-24T22:14:40,205 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=516ead6fe1773c4da8708cdc5713f526, ASSIGN to state=OFFLINE, location=null, table=test, region=516ead6fe1773c4da8708cdc5713f526 to restore RIT 2024-11-24T22:14:40,205 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae, ASSIGN to state=OFFLINE, location=null, table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae to restore RIT 2024-11-24T22:14:40,205 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=84dd509902edc43083bc78fe4b70922c, ASSIGN to state=OFFLINE, location=null, table=test, region=84dd509902edc43083bc78fe4b70922c to restore RIT 2024-11-24T22:14:40,208 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732486510208 2024-11-24T22:14:40,208 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T22:14:40,209 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:40,211 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T22:14:40,212 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T22:14:40,212 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae, ASSIGN 2024-11-24T22:14:40,212 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T22:14:40,212 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:40,216 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e91326d9778777879bc895ecb241af8, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,216 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4279880609a62ccb9e9fb29e79bdf7e7, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,216 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c68a44a11eeaccf8a3c2f49ba1446c1e, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,216 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=698dd9d64bdaf6a81e797d2590ff86f9, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,216 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46b79f5a3db63445091682b5e72f635a, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,217 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=736f44b8f348e2a5a96dfd3abc5e62b2, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,217 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f72c935213180e3ff989af795022b7e4, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,217 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=72a41015f7b88194cc3a08feb66aa4f9, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,217 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94f2fa48145fe3bdeffbfded6a4edd73, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,217 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b2e9ce57ffd0cd61c72602e5f6d9cae, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,217 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b86957c7b81466a818bbff45fac751f, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,217 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=967acf99df098c28d8f837a9f0746a90, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,217 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc147c653b22684b8e41092cd6a0c8eb, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,217 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5160ff1dd0e56b08fbca2ffe581141d4, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,218 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce1f0bcb3026cb1cbbd9e63c21b83f99, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,218 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cca6a66154e5c299fad3a247afbfb5fd, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=5 2024-11-24T22:14:40,218 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83e58b0e0989e11c83283bf7bd41306c, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,218 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c802d82f160c5269119d007388afb85e, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,218 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=84dd509902edc43083bc78fe4b70922c, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,218 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acdab7e426c93b10b34b4d331b21cef0, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,218 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79c2dd6f654fabefe0fc06b8199e8562, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,218 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc9f96fd4c44240a85e6b2c001e46153, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,219 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50d9b38af2d6413e500860133d150aa8, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,219 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=516ead6fe1773c4da8708cdc5713f526, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,219 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79d315a497ec3ad891988cb800bae337, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,219 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed5b70c013e388180062b1513b980a07, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,219 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T22:14:40,219 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732486540219 2024-11-24T22:14:40,219 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732486600219 2024-11-24T22:14:40,219 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-11-24T22:14:40,219 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T22:14:40,219 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T22:14:40,219 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T22:14:40,219 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T22:14:40,220 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T22:14:40,220 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T22:14:40,220 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T22:14:40,221 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T22:14:40,222 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-24T22:14:40,224 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T22:14:40,224 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T22:14:40,224 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T22:14:40,224 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:40,224 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:40,224 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:40,224 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:40,224 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:40,224 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:40,224 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T22:14:40,224 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T22:14:40,224 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:40,224 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T22:14:40,224 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T22:14:40,225 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T22:14:40,226 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=cca6a66154e5c299fad3a247afbfb5fd, REOPEN/MOVE 2024-11-24T22:14:40,227 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true 2024-11-24T22:14:40,227 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:40,227 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:40,227 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T22:14:40,227 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:40,227 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:40,227 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:40,227 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN 2024-11-24T22:14:40,227 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN 2024-11-24T22:14:40,227 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN 2024-11-24T22:14:40,227 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f72c935213180e3ff989af795022b7e4, ASSIGN 2024-11-24T22:14:40,227 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=516ead6fe1773c4da8708cdc5713f526, ASSIGN 2024-11-24T22:14:40,227 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae, ASSIGN 2024-11-24T22:14:40,227 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:40,227 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:40,227 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:40,227 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:40,227 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:40,227 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:40,227 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:40,227 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:40,227 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN 2024-11-24T22:14:40,227 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:40,227 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN 2024-11-24T22:14:40,227 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:40,227 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN 2024-11-24T22:14:40,228 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f72c935213180e3ff989af795022b7e4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:40,228 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f72c935213180e3ff989af795022b7e4, ASSIGN 2024-11-24T22:14:40,228 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=516ead6fe1773c4da8708cdc5713f526, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:40,228 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=516ead6fe1773c4da8708cdc5713f526, ASSIGN 2024-11-24T22:14:40,228 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:40,228 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae, ASSIGN 2024-11-24T22:14:40,228 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-24T22:14:40,231 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=65 2024-11-24T22:14:40,231 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=bd6592c37bb2,42639,1732486469476, table=hbase:meta, region=1588230740 2024-11-24T22:14:40,231 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bd6592c37bb2,42639,1732486469476, state=OPEN 2024-11-24T22:14:40,234 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36279-0x1014c0b79ef0000, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:40,234 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42639-0x1014c0b79ef0003, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:40,234 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38125-0x1014c0b79ef0002, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:40,235 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:40,235 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:40,235 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:40,235 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN to state=OFFLINE, location=null, table=test, region=dc147c653b22684b8e41092cd6a0c8eb to restore RIT 2024-11-24T22:14:40,236 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN to state=OFFLINE, location=null, table=test, region=fc9f96fd4c44240a85e6b2c001e46153 to restore RIT 2024-11-24T22:14:40,236 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN to state=OFFLINE, location=null, table=test, region=698dd9d64bdaf6a81e797d2590ff86f9 to restore RIT 2024-11-24T22:14:40,236 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f72c935213180e3ff989af795022b7e4, ASSIGN to state=OFFLINE, location=null, table=test, region=f72c935213180e3ff989af795022b7e4 to restore RIT 2024-11-24T22:14:40,236 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=516ead6fe1773c4da8708cdc5713f526, ASSIGN to state=OFFLINE, location=null, table=test, region=516ead6fe1773c4da8708cdc5713f526 to restore RIT 2024-11-24T22:14:40,236 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae, ASSIGN to state=OFFLINE, location=null, table=test, region=9b2e9ce57ffd0cd61c72602e5f6d9cae to restore RIT 2024-11-24T22:14:40,240 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732486510240 2024-11-24T22:14:40,240 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T22:14:40,240 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:40,242 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T22:14:40,250 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e91326d9778777879bc895ecb241af8, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,250 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4279880609a62ccb9e9fb29e79bdf7e7, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,250 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c68a44a11eeaccf8a3c2f49ba1446c1e, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,250 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=698dd9d64bdaf6a81e797d2590ff86f9, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,250 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46b79f5a3db63445091682b5e72f635a, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,251 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=736f44b8f348e2a5a96dfd3abc5e62b2, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,251 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f72c935213180e3ff989af795022b7e4, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,251 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=72a41015f7b88194cc3a08feb66aa4f9, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,251 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94f2fa48145fe3bdeffbfded6a4edd73, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,251 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b2e9ce57ffd0cd61c72602e5f6d9cae, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,251 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b86957c7b81466a818bbff45fac751f, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,251 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=967acf99df098c28d8f837a9f0746a90, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,251 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc147c653b22684b8e41092cd6a0c8eb, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,251 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5160ff1dd0e56b08fbca2ffe581141d4, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,251 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce1f0bcb3026cb1cbbd9e63c21b83f99, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cca6a66154e5c299fad3a247afbfb5fd, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=5 2024-11-24T22:14:40,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83e58b0e0989e11c83283bf7bd41306c, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c802d82f160c5269119d007388afb85e, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=84dd509902edc43083bc78fe4b70922c, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acdab7e426c93b10b34b4d331b21cef0, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79c2dd6f654fabefe0fc06b8199e8562, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc9f96fd4c44240a85e6b2c001e46153, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50d9b38af2d6413e500860133d150aa8, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=516ead6fe1773c4da8708cdc5713f526, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79d315a497ec3ad891988cb800bae337, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed5b70c013e388180062b1513b980a07, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,253 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T22:14:40,253 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732486540253 2024-11-24T22:14:40,253 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732486600253 2024-11-24T22:14:40,253 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 12 msec 2024-11-24T22:14:40,260 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T22:14:40,261 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=516ead6fe1773c4da8708cdc5713f526, ASSIGN 2024-11-24T22:14:40,261 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T22:14:40,261 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=516ead6fe1773c4da8708cdc5713f526, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:40,503 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T22:14:40,503 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T22:14:40,503 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T22:14:40,503 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T22:14:40,504 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T22:14:40,504 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T22:14:40,504 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T22:14:40,504 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T22:14:40,505 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-24T22:14:40,506 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T22:14:40,507 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T22:14:40,507 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T22:14:40,507 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:40,507 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:40,507 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:40,507 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:40,507 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:40,507 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T22:14:40,507 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T22:14:40,507 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:40,507 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T22:14:40,507 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T22:14:40,508 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T22:14:40,508 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=cca6a66154e5c299fad3a247afbfb5fd, REOPEN/MOVE 2024-11-24T22:14:40,509 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true 2024-11-24T22:14:40,509 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:40,509 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:40,509 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T22:14:40,509 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:40,509 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:40,509 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:40,509 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN 2024-11-24T22:14:40,509 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN 2024-11-24T22:14:40,509 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN 2024-11-24T22:14:40,509 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f72c935213180e3ff989af795022b7e4, ASSIGN 2024-11-24T22:14:40,509 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=516ead6fe1773c4da8708cdc5713f526, ASSIGN 2024-11-24T22:14:40,509 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:40,510 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:40,510 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:40,510 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:40,510 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:40,510 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:40,510 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:40,510 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:40,510 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN 2024-11-24T22:14:40,510 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:40,510 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN 2024-11-24T22:14:40,510 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:40,510 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN 2024-11-24T22:14:40,510 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f72c935213180e3ff989af795022b7e4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:40,510 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f72c935213180e3ff989af795022b7e4, ASSIGN 2024-11-24T22:14:40,510 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=516ead6fe1773c4da8708cdc5713f526, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:40,510 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=516ead6fe1773c4da8708cdc5713f526, ASSIGN 2024-11-24T22:14:40,510 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-24T22:14:40,511 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=65 2024-11-24T22:14:40,511 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=bd6592c37bb2,42639,1732486469476, table=hbase:meta, region=1588230740 2024-11-24T22:14:40,511 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bd6592c37bb2,42639,1732486469476, state=OPEN 2024-11-24T22:14:40,514 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36279-0x1014c0b79ef0000, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:40,514 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38125-0x1014c0b79ef0002, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:40,514 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42639-0x1014c0b79ef0003, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:40,514 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:40,514 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:40,514 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:40,515 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN to state=OFFLINE, location=null, table=test, region=dc147c653b22684b8e41092cd6a0c8eb to restore RIT 2024-11-24T22:14:40,515 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN to state=OFFLINE, location=null, table=test, region=fc9f96fd4c44240a85e6b2c001e46153 to restore RIT 2024-11-24T22:14:40,515 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN to state=OFFLINE, location=null, table=test, region=698dd9d64bdaf6a81e797d2590ff86f9 to restore RIT 2024-11-24T22:14:40,515 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f72c935213180e3ff989af795022b7e4, ASSIGN to state=OFFLINE, location=null, table=test, region=f72c935213180e3ff989af795022b7e4 to restore RIT 2024-11-24T22:14:40,515 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=516ead6fe1773c4da8708cdc5713f526, ASSIGN to state=OFFLINE, location=null, table=test, region=516ead6fe1773c4da8708cdc5713f526 to restore RIT 2024-11-24T22:14:40,515 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732486510515 2024-11-24T22:14:40,515 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T22:14:40,516 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:40,517 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T22:14:40,518 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T22:14:40,518 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f72c935213180e3ff989af795022b7e4, ASSIGN 2024-11-24T22:14:40,518 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T22:14:40,518 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f72c935213180e3ff989af795022b7e4, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:40,521 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e91326d9778777879bc895ecb241af8, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,522 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4279880609a62ccb9e9fb29e79bdf7e7, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,522 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c68a44a11eeaccf8a3c2f49ba1446c1e, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,522 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=698dd9d64bdaf6a81e797d2590ff86f9, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,522 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46b79f5a3db63445091682b5e72f635a, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,522 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=736f44b8f348e2a5a96dfd3abc5e62b2, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,522 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f72c935213180e3ff989af795022b7e4, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,522 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=72a41015f7b88194cc3a08feb66aa4f9, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,522 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94f2fa48145fe3bdeffbfded6a4edd73, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,522 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b2e9ce57ffd0cd61c72602e5f6d9cae, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,522 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b86957c7b81466a818bbff45fac751f, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,522 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=967acf99df098c28d8f837a9f0746a90, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,522 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc147c653b22684b8e41092cd6a0c8eb, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,522 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5160ff1dd0e56b08fbca2ffe581141d4, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,523 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce1f0bcb3026cb1cbbd9e63c21b83f99, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,523 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cca6a66154e5c299fad3a247afbfb5fd, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=5 2024-11-24T22:14:40,523 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83e58b0e0989e11c83283bf7bd41306c, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,523 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c802d82f160c5269119d007388afb85e, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,523 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=84dd509902edc43083bc78fe4b70922c, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,523 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acdab7e426c93b10b34b4d331b21cef0, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,523 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79c2dd6f654fabefe0fc06b8199e8562, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,523 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc9f96fd4c44240a85e6b2c001e46153, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,523 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50d9b38af2d6413e500860133d150aa8, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,523 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=516ead6fe1773c4da8708cdc5713f526, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,523 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79d315a497ec3ad891988cb800bae337, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,523 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed5b70c013e388180062b1513b980a07, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,523 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T22:14:40,523 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732486540523 2024-11-24T22:14:40,523 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732486600523 2024-11-24T22:14:40,523 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-24T22:14:40,523 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T22:14:40,523 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T22:14:40,523 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T22:14:40,524 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T22:14:40,524 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T22:14:40,524 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T22:14:40,524 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T22:14:40,524 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T22:14:40,525 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-24T22:14:40,526 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T22:14:40,526 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T22:14:40,526 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T22:14:40,526 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:40,526 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:40,526 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:40,527 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:40,527 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T22:14:40,527 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T22:14:40,527 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:40,527 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T22:14:40,527 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T22:14:40,527 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T22:14:40,527 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=cca6a66154e5c299fad3a247afbfb5fd, REOPEN/MOVE 2024-11-24T22:14:40,528 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true 2024-11-24T22:14:40,528 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:40,528 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:40,528 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T22:14:40,528 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:40,528 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:40,528 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:40,528 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN 2024-11-24T22:14:40,528 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN 2024-11-24T22:14:40,528 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN 2024-11-24T22:14:40,528 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f72c935213180e3ff989af795022b7e4, ASSIGN 2024-11-24T22:14:40,528 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:40,528 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:40,528 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:40,529 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:40,529 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:40,529 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:40,529 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:40,529 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:40,529 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN 2024-11-24T22:14:40,529 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:40,529 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN 2024-11-24T22:14:40,529 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:40,529 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN 2024-11-24T22:14:40,529 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f72c935213180e3ff989af795022b7e4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:40,529 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f72c935213180e3ff989af795022b7e4, ASSIGN 2024-11-24T22:14:40,529 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-24T22:14:40,529 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=65 2024-11-24T22:14:40,530 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=bd6592c37bb2,42639,1732486469476, table=hbase:meta, region=1588230740 2024-11-24T22:14:40,530 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bd6592c37bb2,42639,1732486469476, state=OPEN 2024-11-24T22:14:40,533 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42639-0x1014c0b79ef0003, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:40,533 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36279-0x1014c0b79ef0000, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:40,533 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:40,533 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38125-0x1014c0b79ef0002, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:40,533 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:40,533 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:40,536 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN to state=OFFLINE, location=null, table=test, region=dc147c653b22684b8e41092cd6a0c8eb to restore RIT 2024-11-24T22:14:40,536 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN to state=OFFLINE, location=null, table=test, region=fc9f96fd4c44240a85e6b2c001e46153 to restore RIT 2024-11-24T22:14:40,536 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN to state=OFFLINE, location=null, table=test, region=698dd9d64bdaf6a81e797d2590ff86f9 to restore RIT 2024-11-24T22:14:40,536 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f72c935213180e3ff989af795022b7e4, ASSIGN to state=OFFLINE, location=null, table=test, region=f72c935213180e3ff989af795022b7e4 to restore RIT 2024-11-24T22:14:40,544 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732486510544 2024-11-24T22:14:40,544 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T22:14:40,544 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:40,547 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T22:14:40,549 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T22:14:40,549 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN 2024-11-24T22:14:40,549 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T22:14:40,549 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:40,551 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e91326d9778777879bc895ecb241af8, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,551 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4279880609a62ccb9e9fb29e79bdf7e7, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,551 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c68a44a11eeaccf8a3c2f49ba1446c1e, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,551 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=698dd9d64bdaf6a81e797d2590ff86f9, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,552 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46b79f5a3db63445091682b5e72f635a, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,552 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=736f44b8f348e2a5a96dfd3abc5e62b2, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,552 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f72c935213180e3ff989af795022b7e4, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,552 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=72a41015f7b88194cc3a08feb66aa4f9, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,552 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94f2fa48145fe3bdeffbfded6a4edd73, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,552 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b2e9ce57ffd0cd61c72602e5f6d9cae, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,552 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b86957c7b81466a818bbff45fac751f, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,553 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=967acf99df098c28d8f837a9f0746a90, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,553 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc147c653b22684b8e41092cd6a0c8eb, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,553 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5160ff1dd0e56b08fbca2ffe581141d4, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,553 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce1f0bcb3026cb1cbbd9e63c21b83f99, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,553 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cca6a66154e5c299fad3a247afbfb5fd, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=5 2024-11-24T22:14:40,553 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83e58b0e0989e11c83283bf7bd41306c, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,554 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c802d82f160c5269119d007388afb85e, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,555 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=84dd509902edc43083bc78fe4b70922c, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,556 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acdab7e426c93b10b34b4d331b21cef0, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,556 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79c2dd6f654fabefe0fc06b8199e8562, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,556 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc9f96fd4c44240a85e6b2c001e46153, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,556 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50d9b38af2d6413e500860133d150aa8, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,556 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=516ead6fe1773c4da8708cdc5713f526, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,557 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79d315a497ec3ad891988cb800bae337, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,557 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed5b70c013e388180062b1513b980a07, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,557 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T22:14:40,557 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732486540557 2024-11-24T22:14:40,557 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732486600557 2024-11-24T22:14:40,557 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 13 msec 2024-11-24T22:14:40,557 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T22:14:40,557 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T22:14:40,557 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T22:14:40,557 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T22:14:40,558 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T22:14:40,558 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T22:14:40,558 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T22:14:40,558 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T22:14:40,559 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-24T22:14:40,563 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36279 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:40,566 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36279 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:40,569 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T22:14:40,569 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T22:14:40,569 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T22:14:40,569 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:40,569 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:40,569 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:40,569 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T22:14:40,570 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T22:14:40,570 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:40,570 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T22:14:40,570 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T22:14:40,571 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T22:14:40,571 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=cca6a66154e5c299fad3a247afbfb5fd, REOPEN/MOVE 2024-11-24T22:14:40,572 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true 2024-11-24T22:14:40,572 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:40,572 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:40,572 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T22:14:40,572 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:40,572 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:40,572 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:40,572 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN 2024-11-24T22:14:40,572 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN 2024-11-24T22:14:40,572 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN 2024-11-24T22:14:40,572 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:40,573 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:40,573 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:40,573 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:40,573 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:40,573 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:40,573 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:40,573 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:40,573 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN 2024-11-24T22:14:40,573 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:40,573 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN 2024-11-24T22:14:40,573 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:40,573 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN 2024-11-24T22:14:40,573 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 14 msec 2024-11-24T22:14:40,588 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=65 2024-11-24T22:14:40,588 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=bd6592c37bb2,42639,1732486469476, table=hbase:meta, region=1588230740 2024-11-24T22:14:40,589 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bd6592c37bb2,42639,1732486469476, state=OPEN 2024-11-24T22:14:40,594 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42639-0x1014c0b79ef0003, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:40,594 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:40,594 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36279-0x1014c0b79ef0000, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:40,594 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:40,594 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38125-0x1014c0b79ef0002, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:40,595 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:40,597 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN to state=OFFLINE, location=null, table=test, region=dc147c653b22684b8e41092cd6a0c8eb to restore RIT 2024-11-24T22:14:40,598 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN to state=OFFLINE, location=null, table=test, region=fc9f96fd4c44240a85e6b2c001e46153 to restore RIT 2024-11-24T22:14:40,599 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=698dd9d64bdaf6a81e797d2590ff86f9, ASSIGN to state=OFFLINE, location=null, table=test, region=698dd9d64bdaf6a81e797d2590ff86f9 to restore RIT 2024-11-24T22:14:40,632 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732486510632 2024-11-24T22:14:40,632 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T22:14:40,634 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:40,635 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T22:14:40,640 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T22:14:40,640 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN 2024-11-24T22:14:40,640 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T22:14:40,640 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e91326d9778777879bc895ecb241af8, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,640 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4279880609a62ccb9e9fb29e79bdf7e7, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,640 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c68a44a11eeaccf8a3c2f49ba1446c1e, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,640 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:40,640 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=698dd9d64bdaf6a81e797d2590ff86f9, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,641 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46b79f5a3db63445091682b5e72f635a, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,641 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=736f44b8f348e2a5a96dfd3abc5e62b2, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,641 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f72c935213180e3ff989af795022b7e4, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,641 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=72a41015f7b88194cc3a08feb66aa4f9, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,641 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94f2fa48145fe3bdeffbfded6a4edd73, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,641 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b2e9ce57ffd0cd61c72602e5f6d9cae, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,641 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b86957c7b81466a818bbff45fac751f, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,642 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=967acf99df098c28d8f837a9f0746a90, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,642 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc147c653b22684b8e41092cd6a0c8eb, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,642 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5160ff1dd0e56b08fbca2ffe581141d4, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,642 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce1f0bcb3026cb1cbbd9e63c21b83f99, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,642 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cca6a66154e5c299fad3a247afbfb5fd, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=5 2024-11-24T22:14:40,642 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83e58b0e0989e11c83283bf7bd41306c, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,643 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c802d82f160c5269119d007388afb85e, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,643 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=84dd509902edc43083bc78fe4b70922c, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,643 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acdab7e426c93b10b34b4d331b21cef0, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,643 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79c2dd6f654fabefe0fc06b8199e8562, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,643 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc9f96fd4c44240a85e6b2c001e46153, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,643 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50d9b38af2d6413e500860133d150aa8, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,643 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=516ead6fe1773c4da8708cdc5713f526, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,643 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79d315a497ec3ad891988cb800bae337, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,644 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed5b70c013e388180062b1513b980a07, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,644 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T22:14:40,644 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732486540644 2024-11-24T22:14:40,644 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732486600644 2024-11-24T22:14:40,644 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 11 msec 2024-11-24T22:14:40,644 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T22:14:40,644 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T22:14:40,644 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T22:14:40,644 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T22:14:40,644 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T22:14:40,644 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T22:14:40,644 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T22:14:40,646 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T22:14:40,649 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 3 msec 2024-11-24T22:14:40,651 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T22:14:40,651 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T22:14:40,651 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T22:14:40,651 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:40,651 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:40,651 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T22:14:40,651 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T22:14:40,651 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:40,651 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T22:14:40,651 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T22:14:40,652 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T22:14:40,652 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=cca6a66154e5c299fad3a247afbfb5fd, REOPEN/MOVE 2024-11-24T22:14:40,654 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true 2024-11-24T22:14:40,654 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:40,654 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:40,654 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T22:14:40,654 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:40,654 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:40,654 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:40,654 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN 2024-11-24T22:14:40,654 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN 2024-11-24T22:14:40,654 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:40,654 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:40,654 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:40,654 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:40,654 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:40,654 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:40,654 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:40,655 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:40,655 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN 2024-11-24T22:14:40,655 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:40,655 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN 2024-11-24T22:14:40,655 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-24T22:14:40,657 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=65 2024-11-24T22:14:40,657 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=bd6592c37bb2,42639,1732486469476, table=hbase:meta, region=1588230740 2024-11-24T22:14:40,657 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bd6592c37bb2,42639,1732486469476, state=OPEN 2024-11-24T22:14:40,666 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38125-0x1014c0b79ef0002, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:40,666 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36279-0x1014c0b79ef0000, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:40,666 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:40,666 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:40,667 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42639-0x1014c0b79ef0003, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:40,667 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:40,667 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36279 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:40,668 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN to state=OFFLINE, location=null, table=test, region=dc147c653b22684b8e41092cd6a0c8eb to restore RIT 2024-11-24T22:14:40,668 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fc9f96fd4c44240a85e6b2c001e46153, ASSIGN to state=OFFLINE, location=null, table=test, region=fc9f96fd4c44240a85e6b2c001e46153 to restore RIT 2024-11-24T22:14:40,674 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732486510674 2024-11-24T22:14:40,674 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T22:14:40,677 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:40,679 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T22:14:40,683 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e91326d9778777879bc895ecb241af8, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,683 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4279880609a62ccb9e9fb29e79bdf7e7, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,683 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c68a44a11eeaccf8a3c2f49ba1446c1e, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,683 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=698dd9d64bdaf6a81e797d2590ff86f9, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,683 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46b79f5a3db63445091682b5e72f635a, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,683 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=736f44b8f348e2a5a96dfd3abc5e62b2, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,683 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T22:14:40,684 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f72c935213180e3ff989af795022b7e4, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,684 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN 2024-11-24T22:14:40,684 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T22:14:40,684 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=72a41015f7b88194cc3a08feb66aa4f9, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,684 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94f2fa48145fe3bdeffbfded6a4edd73, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,684 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b2e9ce57ffd0cd61c72602e5f6d9cae, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,684 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b86957c7b81466a818bbff45fac751f, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,684 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=967acf99df098c28d8f837a9f0746a90, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,684 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc147c653b22684b8e41092cd6a0c8eb, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,684 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5160ff1dd0e56b08fbca2ffe581141d4, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,684 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce1f0bcb3026cb1cbbd9e63c21b83f99, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,684 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cca6a66154e5c299fad3a247afbfb5fd, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=5 2024-11-24T22:14:40,685 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83e58b0e0989e11c83283bf7bd41306c, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,685 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c802d82f160c5269119d007388afb85e, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,685 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:40,685 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=84dd509902edc43083bc78fe4b70922c, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,685 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acdab7e426c93b10b34b4d331b21cef0, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,685 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79c2dd6f654fabefe0fc06b8199e8562, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,685 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc9f96fd4c44240a85e6b2c001e46153, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,685 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50d9b38af2d6413e500860133d150aa8, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,685 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=516ead6fe1773c4da8708cdc5713f526, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,686 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79d315a497ec3ad891988cb800bae337, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,686 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed5b70c013e388180062b1513b980a07, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,686 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T22:14:40,686 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732486540686 2024-11-24T22:14:40,686 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732486600686 2024-11-24T22:14:40,686 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 11 msec 2024-11-24T22:14:40,686 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T22:14:40,686 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T22:14:40,686 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T22:14:40,686 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T22:14:40,686 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T22:14:40,687 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T22:14:40,687 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T22:14:40,687 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T22:14:40,689 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 2 msec 2024-11-24T22:14:40,691 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T22:14:40,691 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T22:14:40,691 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T22:14:40,691 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:40,691 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T22:14:40,691 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T22:14:40,691 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:40,691 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T22:14:40,691 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T22:14:40,692 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T22:14:40,692 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=cca6a66154e5c299fad3a247afbfb5fd, REOPEN/MOVE 2024-11-24T22:14:40,693 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true 2024-11-24T22:14:40,694 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:40,694 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:40,694 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T22:14:40,694 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:40,694 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:40,694 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:40,694 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN 2024-11-24T22:14:40,694 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:40,694 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:40,694 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:40,694 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:40,694 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:40,694 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:40,694 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:40,694 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:40,694 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN 2024-11-24T22:14:40,694 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-24T22:14:40,699 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=65 2024-11-24T22:14:40,699 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=bd6592c37bb2,42639,1732486469476, table=hbase:meta, region=1588230740 2024-11-24T22:14:40,699 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bd6592c37bb2,42639,1732486469476, state=OPEN 2024-11-24T22:14:40,701 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38125-0x1014c0b79ef0002, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:40,701 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42639-0x1014c0b79ef0003, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:40,701 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36279-0x1014c0b79ef0000, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:40,702 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:40,702 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:40,702 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:40,703 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dc147c653b22684b8e41092cd6a0c8eb, ASSIGN to state=OFFLINE, location=null, table=test, region=dc147c653b22684b8e41092cd6a0c8eb to restore RIT 2024-11-24T22:14:40,721 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732486510721 2024-11-24T22:14:40,721 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T22:14:40,723 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:40,725 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T22:14:40,726 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T22:14:40,726 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:40,726 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T22:14:40,727 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:40,728 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e91326d9778777879bc895ecb241af8, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,728 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4279880609a62ccb9e9fb29e79bdf7e7, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,729 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c68a44a11eeaccf8a3c2f49ba1446c1e, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,729 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=698dd9d64bdaf6a81e797d2590ff86f9, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,729 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46b79f5a3db63445091682b5e72f635a, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,729 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=736f44b8f348e2a5a96dfd3abc5e62b2, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,729 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f72c935213180e3ff989af795022b7e4, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,729 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=72a41015f7b88194cc3a08feb66aa4f9, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,729 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94f2fa48145fe3bdeffbfded6a4edd73, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,729 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b2e9ce57ffd0cd61c72602e5f6d9cae, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,729 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b86957c7b81466a818bbff45fac751f, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,730 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=967acf99df098c28d8f837a9f0746a90, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,730 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc147c653b22684b8e41092cd6a0c8eb, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,730 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5160ff1dd0e56b08fbca2ffe581141d4, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,730 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce1f0bcb3026cb1cbbd9e63c21b83f99, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,730 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cca6a66154e5c299fad3a247afbfb5fd, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=5 2024-11-24T22:14:40,730 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83e58b0e0989e11c83283bf7bd41306c, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,730 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c802d82f160c5269119d007388afb85e, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,730 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=84dd509902edc43083bc78fe4b70922c, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,730 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acdab7e426c93b10b34b4d331b21cef0, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,730 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79c2dd6f654fabefe0fc06b8199e8562, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,731 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc9f96fd4c44240a85e6b2c001e46153, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,731 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50d9b38af2d6413e500860133d150aa8, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,731 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=516ead6fe1773c4da8708cdc5713f526, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,731 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79d315a497ec3ad891988cb800bae337, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,731 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed5b70c013e388180062b1513b980a07, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,731 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T22:14:40,731 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732486540731 2024-11-24T22:14:40,731 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732486600731 2024-11-24T22:14:40,731 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-24T22:14:40,731 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T22:14:40,731 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T22:14:40,731 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T22:14:40,731 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T22:14:40,732 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T22:14:40,732 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T22:14:40,732 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T22:14:40,732 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T22:14:40,733 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-24T22:14:40,735 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T22:14:40,735 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T22:14:40,735 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T22:14:40,735 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T22:14:40,735 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T22:14:40,735 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:40,735 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T22:14:40,735 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T22:14:40,736 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T22:14:40,736 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=cca6a66154e5c299fad3a247afbfb5fd, REOPEN/MOVE 2024-11-24T22:14:40,737 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true 2024-11-24T22:14:40,737 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:40,737 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:40,737 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T22:14:40,737 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:40,737 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:40,738 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:40,738 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:40,740 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:40,740 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:40,740 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:40,740 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:40,740 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:40,740 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:40,741 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-24T22:14:40,745 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=65 2024-11-24T22:14:40,745 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=bd6592c37bb2,42639,1732486469476, table=hbase:meta, region=1588230740 2024-11-24T22:14:40,745 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bd6592c37bb2,42639,1732486469476, state=OPEN 2024-11-24T22:14:40,747 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38125-0x1014c0b79ef0002, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:40,747 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36279-0x1014c0b79ef0000, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:40,747 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42639-0x1014c0b79ef0003, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:40,747 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:40,747 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:40,747 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:40,754 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732486510754 2024-11-24T22:14:40,754 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T22:14:40,754 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:40,764 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T22:14:40,760 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e91326d9778777879bc895ecb241af8, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,764 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4279880609a62ccb9e9fb29e79bdf7e7, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,765 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c68a44a11eeaccf8a3c2f49ba1446c1e, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,765 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=698dd9d64bdaf6a81e797d2590ff86f9, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,765 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46b79f5a3db63445091682b5e72f635a, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,765 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=736f44b8f348e2a5a96dfd3abc5e62b2, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,765 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f72c935213180e3ff989af795022b7e4, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,765 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=72a41015f7b88194cc3a08feb66aa4f9, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,765 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94f2fa48145fe3bdeffbfded6a4edd73, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,765 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b2e9ce57ffd0cd61c72602e5f6d9cae, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,765 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b86957c7b81466a818bbff45fac751f, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,765 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=967acf99df098c28d8f837a9f0746a90, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc147c653b22684b8e41092cd6a0c8eb, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5160ff1dd0e56b08fbca2ffe581141d4, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce1f0bcb3026cb1cbbd9e63c21b83f99, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cca6a66154e5c299fad3a247afbfb5fd, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=5 2024-11-24T22:14:40,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83e58b0e0989e11c83283bf7bd41306c, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c802d82f160c5269119d007388afb85e, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=84dd509902edc43083bc78fe4b70922c, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acdab7e426c93b10b34b4d331b21cef0, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79c2dd6f654fabefe0fc06b8199e8562, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc9f96fd4c44240a85e6b2c001e46153, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50d9b38af2d6413e500860133d150aa8, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=516ead6fe1773c4da8708cdc5713f526, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79d315a497ec3ad891988cb800bae337, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:40,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed5b70c013e388180062b1513b980a07, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:40,766 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T22:14:40,767 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732486540766 2024-11-24T22:14:40,767 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732486600767 2024-11-24T22:14:40,767 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 12 msec 2024-11-24T22:14:40,767 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T22:14:40,767 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:40,767 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T22:14:40,767 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:41,017 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T22:14:41,018 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T22:14:41,018 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T22:14:41,018 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T22:14:41,019 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T22:14:41,019 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T22:14:41,019 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T22:14:41,020 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T22:14:41,021 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-24T22:14:41,023 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T22:14:41,023 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T22:14:41,024 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T22:14:41,024 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T22:14:41,024 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:41,024 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T22:14:41,024 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T22:14:41,024 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T22:14:41,025 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=cca6a66154e5c299fad3a247afbfb5fd, REOPEN/MOVE 2024-11-24T22:14:41,025 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true 2024-11-24T22:14:41,026 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:41,026 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:41,026 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T22:14:41,026 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:41,026 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:41,026 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:41,026 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:41,026 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:41,026 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:41,026 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:41,026 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.1732486471450, worker=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:41,026 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-24T22:14:41,027 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=65 2024-11-24T22:14:41,027 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=bd6592c37bb2,42639,1732486469476, table=hbase:meta, region=1588230740 2024-11-24T22:14:41,027 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bd6592c37bb2,42639,1732486469476, state=OPEN 2024-11-24T22:14:41,029 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38125-0x1014c0b79ef0002, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:41,029 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36279-0x1014c0b79ef0000, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:41,029 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42639-0x1014c0b79ef0003, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:41,029 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:41,029 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:41,029 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:41,040 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732486511040 2024-11-24T22:14:41,040 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T22:14:41,041 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:41,044 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T22:14:41,046 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T22:14:41,046 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:41,046 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T22:14:41,046 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bd6592c37bb2,42639,1732486469476 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:41,047 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e91326d9778777879bc895ecb241af8, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,047 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4279880609a62ccb9e9fb29e79bdf7e7, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,047 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c68a44a11eeaccf8a3c2f49ba1446c1e, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,047 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=698dd9d64bdaf6a81e797d2590ff86f9, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,048 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46b79f5a3db63445091682b5e72f635a, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,048 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=736f44b8f348e2a5a96dfd3abc5e62b2, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,048 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f72c935213180e3ff989af795022b7e4, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,048 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=72a41015f7b88194cc3a08feb66aa4f9, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,048 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94f2fa48145fe3bdeffbfded6a4edd73, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,048 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b2e9ce57ffd0cd61c72602e5f6d9cae, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,048 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b86957c7b81466a818bbff45fac751f, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,048 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=967acf99df098c28d8f837a9f0746a90, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,048 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc147c653b22684b8e41092cd6a0c8eb, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,048 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5160ff1dd0e56b08fbca2ffe581141d4, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,048 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce1f0bcb3026cb1cbbd9e63c21b83f99, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,049 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cca6a66154e5c299fad3a247afbfb5fd, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=5 2024-11-24T22:14:41,049 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83e58b0e0989e11c83283bf7bd41306c, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,049 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c802d82f160c5269119d007388afb85e, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,049 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=84dd509902edc43083bc78fe4b70922c, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,049 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acdab7e426c93b10b34b4d331b21cef0, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,049 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79c2dd6f654fabefe0fc06b8199e8562, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,049 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc9f96fd4c44240a85e6b2c001e46153, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,049 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50d9b38af2d6413e500860133d150aa8, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,049 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=516ead6fe1773c4da8708cdc5713f526, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,050 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79d315a497ec3ad891988cb800bae337, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,050 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed5b70c013e388180062b1513b980a07, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,050 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T22:14:41,050 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732486541050 2024-11-24T22:14:41,050 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732486601050 2024-11-24T22:14:41,050 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-24T22:14:41,050 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T22:14:41,050 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T22:14:41,050 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T22:14:41,050 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T22:14:41,050 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T22:14:41,050 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T22:14:41,050 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T22:14:41,051 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T22:14:41,052 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-24T22:14:41,054 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T22:14:41,054 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T22:14:41,054 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T22:14:41,054 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:41,054 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-24T22:14:41,054 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T22:14:41,055 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T22:14:41,055 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=cca6a66154e5c299fad3a247afbfb5fd, REOPEN/MOVE 2024-11-24T22:14:41,056 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true 2024-11-24T22:14:41,056 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:41,056 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:41,056 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T22:14:41,056 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:41,056 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:41,056 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:41,056 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:41,056 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:41,056 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bd6592c37bb2,42639,1732486469476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:41,056 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-24T22:14:41,073 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=65 2024-11-24T22:14:41,073 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=bd6592c37bb2,42639,1732486469476, table=hbase:meta, region=1588230740 2024-11-24T22:14:41,073 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bd6592c37bb2,42639,1732486469476, state=OPEN 2024-11-24T22:14:41,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42639-0x1014c0b79ef0003, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:41,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36279-0x1014c0b79ef0000, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:41,077 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:41,077 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:41,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38125-0x1014c0b79ef0002, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:41,077 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:41,084 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732486511084 2024-11-24T22:14:41,084 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T22:14:41,085 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:41,087 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T22:14:41,089 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T22:14:41,089 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T22:14:41,089 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T22:14:41,089 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:41,091 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e91326d9778777879bc895ecb241af8, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,091 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4279880609a62ccb9e9fb29e79bdf7e7, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,091 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c68a44a11eeaccf8a3c2f49ba1446c1e, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,091 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=698dd9d64bdaf6a81e797d2590ff86f9, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,091 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46b79f5a3db63445091682b5e72f635a, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=736f44b8f348e2a5a96dfd3abc5e62b2, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f72c935213180e3ff989af795022b7e4, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=72a41015f7b88194cc3a08feb66aa4f9, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94f2fa48145fe3bdeffbfded6a4edd73, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b2e9ce57ffd0cd61c72602e5f6d9cae, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b86957c7b81466a818bbff45fac751f, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=967acf99df098c28d8f837a9f0746a90, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc147c653b22684b8e41092cd6a0c8eb, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5160ff1dd0e56b08fbca2ffe581141d4, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,093 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce1f0bcb3026cb1cbbd9e63c21b83f99, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,093 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cca6a66154e5c299fad3a247afbfb5fd, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=5 2024-11-24T22:14:41,093 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83e58b0e0989e11c83283bf7bd41306c, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,093 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c802d82f160c5269119d007388afb85e, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,093 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=84dd509902edc43083bc78fe4b70922c, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,093 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acdab7e426c93b10b34b4d331b21cef0, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,093 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79c2dd6f654fabefe0fc06b8199e8562, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,093 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc9f96fd4c44240a85e6b2c001e46153, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,093 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50d9b38af2d6413e500860133d150aa8, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,094 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=516ead6fe1773c4da8708cdc5713f526, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,094 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79d315a497ec3ad891988cb800bae337, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,094 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed5b70c013e388180062b1513b980a07, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,094 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T22:14:41,094 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732486541094 2024-11-24T22:14:41,094 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732486601094 2024-11-24T22:14:41,094 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-24T22:14:41,094 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T22:14:41,094 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T22:14:41,094 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T22:14:41,094 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T22:14:41,095 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T22:14:41,095 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T22:14:41,095 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T22:14:41,096 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T22:14:41,097 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-24T22:14:41,098 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T22:14:41,098 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T22:14:41,099 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T22:14:41,099 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-24T22:14:41,099 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T22:14:41,099 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T22:14:41,100 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=cca6a66154e5c299fad3a247afbfb5fd, REOPEN/MOVE 2024-11-24T22:14:41,100 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true 2024-11-24T22:14:41,100 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:41,100 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:41,101 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T22:14:41,101 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:41,101 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:41,101 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:41,101 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:41,101 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-24T22:14:41,102 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=65 2024-11-24T22:14:41,102 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=bd6592c37bb2,42639,1732486469476, table=hbase:meta, region=1588230740 2024-11-24T22:14:41,102 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bd6592c37bb2,42639,1732486469476, state=OPEN 2024-11-24T22:14:41,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38125-0x1014c0b79ef0002, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:41,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36279-0x1014c0b79ef0000, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:41,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42639-0x1014c0b79ef0003, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:41,104 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:41,104 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:41,104 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:41,112 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732486511112 2024-11-24T22:14:41,112 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T22:14:41,115 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:41,117 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T22:14:41,119 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e91326d9778777879bc895ecb241af8, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,119 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4279880609a62ccb9e9fb29e79bdf7e7, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,119 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c68a44a11eeaccf8a3c2f49ba1446c1e, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,119 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=698dd9d64bdaf6a81e797d2590ff86f9, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,119 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46b79f5a3db63445091682b5e72f635a, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,119 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=736f44b8f348e2a5a96dfd3abc5e62b2, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,120 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f72c935213180e3ff989af795022b7e4, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,120 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=72a41015f7b88194cc3a08feb66aa4f9, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,120 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94f2fa48145fe3bdeffbfded6a4edd73, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,120 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b2e9ce57ffd0cd61c72602e5f6d9cae, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,120 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b86957c7b81466a818bbff45fac751f, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,120 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=967acf99df098c28d8f837a9f0746a90, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,120 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc147c653b22684b8e41092cd6a0c8eb, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,120 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5160ff1dd0e56b08fbca2ffe581141d4, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,120 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce1f0bcb3026cb1cbbd9e63c21b83f99, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,121 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cca6a66154e5c299fad3a247afbfb5fd, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=5 2024-11-24T22:14:41,121 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83e58b0e0989e11c83283bf7bd41306c, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,121 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c802d82f160c5269119d007388afb85e, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,121 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=84dd509902edc43083bc78fe4b70922c, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,121 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acdab7e426c93b10b34b4d331b21cef0, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,121 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79c2dd6f654fabefe0fc06b8199e8562, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,121 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc9f96fd4c44240a85e6b2c001e46153, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,122 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50d9b38af2d6413e500860133d150aa8, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,122 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=516ead6fe1773c4da8708cdc5713f526, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,122 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79d315a497ec3ad891988cb800bae337, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,122 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed5b70c013e388180062b1513b980a07, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,122 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T22:14:41,122 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732486541122 2024-11-24T22:14:41,122 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732486601122 2024-11-24T22:14:41,122 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-11-24T22:14:41,122 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T22:14:41,122 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:41,122 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T22:14:41,123 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:41,372 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T22:14:41,372 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T22:14:41,373 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T22:14:41,373 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T22:14:41,373 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T22:14:41,373 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T22:14:41,373 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T22:14:41,374 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T22:14:41,375 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-24T22:14:41,376 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T22:14:41,376 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T22:14:41,376 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-24T22:14:41,376 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T22:14:41,377 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T22:14:41,377 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=cca6a66154e5c299fad3a247afbfb5fd, REOPEN/MOVE 2024-11-24T22:14:41,378 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true 2024-11-24T22:14:41,378 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:41,378 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:41,378 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:41,378 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:41,378 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:41,378 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-24T22:14:41,379 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=65 2024-11-24T22:14:41,379 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=bd6592c37bb2,42639,1732486469476, table=hbase:meta, region=1588230740 2024-11-24T22:14:41,379 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bd6592c37bb2,42639,1732486469476, state=OPEN 2024-11-24T22:14:41,381 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38125-0x1014c0b79ef0002, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:41,381 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36279-0x1014c0b79ef0000, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:41,381 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42639-0x1014c0b79ef0003, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:41,381 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:41,381 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:41,381 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:41,384 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732486511384 2024-11-24T22:14:41,384 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T22:14:41,387 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:41,391 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T22:14:41,392 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e91326d9778777879bc895ecb241af8, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,392 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4279880609a62ccb9e9fb29e79bdf7e7, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,392 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c68a44a11eeaccf8a3c2f49ba1446c1e, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,393 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=698dd9d64bdaf6a81e797d2590ff86f9, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,393 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46b79f5a3db63445091682b5e72f635a, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,393 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=736f44b8f348e2a5a96dfd3abc5e62b2, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,393 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f72c935213180e3ff989af795022b7e4, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,393 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=72a41015f7b88194cc3a08feb66aa4f9, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,393 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94f2fa48145fe3bdeffbfded6a4edd73, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,393 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b2e9ce57ffd0cd61c72602e5f6d9cae, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,393 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b86957c7b81466a818bbff45fac751f, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,393 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=967acf99df098c28d8f837a9f0746a90, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,393 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc147c653b22684b8e41092cd6a0c8eb, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,394 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5160ff1dd0e56b08fbca2ffe581141d4, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,394 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce1f0bcb3026cb1cbbd9e63c21b83f99, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,394 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cca6a66154e5c299fad3a247afbfb5fd, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=5 2024-11-24T22:14:41,394 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T22:14:41,394 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83e58b0e0989e11c83283bf7bd41306c, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,394 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:41,394 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T22:14:41,394 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c802d82f160c5269119d007388afb85e, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,394 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=84dd509902edc43083bc78fe4b70922c, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,394 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acdab7e426c93b10b34b4d331b21cef0, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,394 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79c2dd6f654fabefe0fc06b8199e8562, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,394 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc9f96fd4c44240a85e6b2c001e46153, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,395 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50d9b38af2d6413e500860133d150aa8, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,395 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=516ead6fe1773c4da8708cdc5713f526, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,395 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79d315a497ec3ad891988cb800bae337, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,395 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed5b70c013e388180062b1513b980a07, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,395 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T22:14:41,395 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732486541395 2024-11-24T22:14:41,395 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732486601395 2024-11-24T22:14:41,396 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 11 msec 2024-11-24T22:14:41,396 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T22:14:41,396 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T22:14:41,396 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T22:14:41,396 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T22:14:41,394 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:41,405 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T22:14:41,405 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T22:14:41,405 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T22:14:41,405 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T22:14:41,406 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-24T22:14:41,407 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T22:14:41,407 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-24T22:14:41,408 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T22:14:41,408 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T22:14:41,409 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=cca6a66154e5c299fad3a247afbfb5fd, REOPEN/MOVE 2024-11-24T22:14:41,409 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true 2024-11-24T22:14:41,409 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:41,409 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:41,409 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure bd6592c37bb2%2C41619%2C1732486469269.meta.1732486471945.meta, worker=bd6592c37bb2,38125,1732486469405 didn't hold the lock before restarting, skip acquiring lock. 2024-11-24T22:14:41,409 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-24T22:14:41,421 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=65 2024-11-24T22:14:41,421 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=bd6592c37bb2,42639,1732486469476, table=hbase:meta, region=1588230740 2024-11-24T22:14:41,421 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bd6592c37bb2,42639,1732486469476, state=OPEN 2024-11-24T22:14:41,423 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42639-0x1014c0b79ef0003, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:41,423 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:41,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38125-0x1014c0b79ef0002, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:41,424 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:41,424 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36279 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:41,425 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36279-0x1014c0b79ef0000, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:41,425 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:41,430 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36279 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:41,476 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732486511476 2024-11-24T22:14:41,476 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T22:14:41,477 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:41,479 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T22:14:41,480 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-24T22:14:41,480 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true 2024-11-24T22:14:41,480 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T22:14:41,480 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1604) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:41,483 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e91326d9778777879bc895ecb241af8, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,483 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4279880609a62ccb9e9fb29e79bdf7e7, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,484 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c68a44a11eeaccf8a3c2f49ba1446c1e, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,484 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=698dd9d64bdaf6a81e797d2590ff86f9, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,484 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46b79f5a3db63445091682b5e72f635a, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,484 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=736f44b8f348e2a5a96dfd3abc5e62b2, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,484 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f72c935213180e3ff989af795022b7e4, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,484 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=72a41015f7b88194cc3a08feb66aa4f9, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,484 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94f2fa48145fe3bdeffbfded6a4edd73, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,484 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b2e9ce57ffd0cd61c72602e5f6d9cae, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,484 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b86957c7b81466a818bbff45fac751f, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,484 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=967acf99df098c28d8f837a9f0746a90, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,484 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc147c653b22684b8e41092cd6a0c8eb, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,484 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5160ff1dd0e56b08fbca2ffe581141d4, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,484 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce1f0bcb3026cb1cbbd9e63c21b83f99, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,485 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cca6a66154e5c299fad3a247afbfb5fd, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=5 2024-11-24T22:14:41,485 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83e58b0e0989e11c83283bf7bd41306c, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,485 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c802d82f160c5269119d007388afb85e, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,485 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=84dd509902edc43083bc78fe4b70922c, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,485 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acdab7e426c93b10b34b4d331b21cef0, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,485 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79c2dd6f654fabefe0fc06b8199e8562, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,485 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc9f96fd4c44240a85e6b2c001e46153, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,485 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50d9b38af2d6413e500860133d150aa8, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,485 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=516ead6fe1773c4da8708cdc5713f526, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,485 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79d315a497ec3ad891988cb800bae337, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,485 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed5b70c013e388180062b1513b980a07, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,485 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T22:14:41,485 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732486541485 2024-11-24T22:14:41,485 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732486601485 2024-11-24T22:14:41,486 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-24T22:14:41,486 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-24T22:14:41,486 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T22:14:41,486 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T22:14:41,486 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T22:14:41,486 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-24T22:14:41,486 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-24T22:14:41,486 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-24T22:14:41,486 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T22:14:41,487 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-24T22:14:41,489 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-24T22:14:41,489 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T22:14:41,490 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-24T22:14:41,490 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=cca6a66154e5c299fad3a247afbfb5fd, REOPEN/MOVE 2024-11-24T22:14:41,491 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true 2024-11-24T22:14:41,491 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-24T22:14:41,491 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-24T22:14:41,501 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=65 2024-11-24T22:14:41,501 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=bd6592c37bb2,42639,1732486469476, table=hbase:meta, region=1588230740 2024-11-24T22:14:41,501 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bd6592c37bb2,42639,1732486469476, state=OPEN 2024-11-24T22:14:41,506 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38125-0x1014c0b79ef0002, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:41,506 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42639-0x1014c0b79ef0003, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:41,507 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36279-0x1014c0b79ef0000, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T22:14:41,507 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:41,507 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:41,507 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T22:14:41,513 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732486511513 2024-11-24T22:14:41,513 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T22:14:41,520 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4e91326d9778777879bc895ecb241af8, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,520 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4279880609a62ccb9e9fb29e79bdf7e7, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,520 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c68a44a11eeaccf8a3c2f49ba1446c1e, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,520 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=698dd9d64bdaf6a81e797d2590ff86f9, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,520 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=46b79f5a3db63445091682b5e72f635a, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,520 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=736f44b8f348e2a5a96dfd3abc5e62b2, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,521 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f72c935213180e3ff989af795022b7e4, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,521 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=72a41015f7b88194cc3a08feb66aa4f9, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,521 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=94f2fa48145fe3bdeffbfded6a4edd73, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,521 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b2e9ce57ffd0cd61c72602e5f6d9cae, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,521 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b86957c7b81466a818bbff45fac751f, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,521 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=967acf99df098c28d8f837a9f0746a90, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,521 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dc147c653b22684b8e41092cd6a0c8eb, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,521 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5160ff1dd0e56b08fbca2ffe581141d4, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,521 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce1f0bcb3026cb1cbbd9e63c21b83f99, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,521 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cca6a66154e5c299fad3a247afbfb5fd, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=5 2024-11-24T22:14:41,521 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83e58b0e0989e11c83283bf7bd41306c, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,521 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c802d82f160c5269119d007388afb85e, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,522 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=84dd509902edc43083bc78fe4b70922c, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,522 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=acdab7e426c93b10b34b4d331b21cef0, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,522 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79c2dd6f654fabefe0fc06b8199e8562, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,522 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fc9f96fd4c44240a85e6b2c001e46153, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,522 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50d9b38af2d6413e500860133d150aa8, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,522 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=516ead6fe1773c4da8708cdc5713f526, regionState=OPENING, lastHost=bd6592c37bb2,41619,1732486469269, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,522 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=79d315a497ec3ad891988cb800bae337, regionState=OPEN, lastHost=bd6592c37bb2,38125,1732486469405, regionLocation=bd6592c37bb2,38125,1732486469405, openSeqNum=2 2024-11-24T22:14:41,522 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed5b70c013e388180062b1513b980a07, regionState=OPEN, lastHost=bd6592c37bb2,42639,1732486469476, regionLocation=bd6592c37bb2,42639,1732486469476, openSeqNum=2 2024-11-24T22:14:41,522 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T22:14:41,522 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732486541522 2024-11-24T22:14:41,522 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732486601522 2024-11-24T22:14:41,522 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-24T22:14:41,513 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-24T22:14:41,534 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-24T22:14:41,536 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=60, state=ROLLEDBACK, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 84dd509902edc43083bc78fe4b70922c, server=bd6592c37bb2,38125,1732486469405:java.lang.RuntimeException: inject code bug; ServerCrashProcedure bd6592c37bb2,41619,1732486469269, splitWal=true, meta=true exec-time=5.0820 sec 2024-11-24T22:14:41,796 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.assignment.TestRollbackSCP#testFailAndRollback Thread=347 (was 381), OpenFileDescriptor=621 (was 646), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=506 (was 445) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6366 (was 6451) 2024-11-24T22:14:41,796 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-24T22:14:41,796 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T22:14:41,796 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T22:14:41,796 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:41,797 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:41,797 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T22:14:41,797 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-24T22:14:41,797 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1096963063, stopped=false 2024-11-24T22:14:41,798 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=bd6592c37bb2,36279,1732486468255 2024-11-24T22:14:41,800 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42639-0x1014c0b79ef0003, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T22:14:41,800 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42639-0x1014c0b79ef0003, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T22:14:41,800 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38125-0x1014c0b79ef0002, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T22:14:41,800 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38125-0x1014c0b79ef0002, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T22:14:41,800 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36279-0x1014c0b79ef0000, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T22:14:41,800 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36279-0x1014c0b79ef0000, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T22:14:41,800 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T22:14:41,801 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T22:14:41,801 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42639-0x1014c0b79ef0003, quorum=127.0.0.1:56583, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T22:14:41,801 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T22:14:41,801 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:41,802 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'bd6592c37bb2,38125,1732486469405' ***** 2024-11-24T22:14:41,802 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-24T22:14:41,802 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'bd6592c37bb2,42639,1732486469476' ***** 2024-11-24T22:14:41,802 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-24T22:14:41,802 INFO [RS:1;bd6592c37bb2:38125 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T22:14:41,802 INFO [RS:1;bd6592c37bb2:38125 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T22:14:41,802 INFO [RS:2;bd6592c37bb2:42639 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T22:14:41,802 INFO [RS:1;bd6592c37bb2:38125 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T22:14:41,802 INFO [RS:2;bd6592c37bb2:42639 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T22:14:41,802 INFO [RS:2;bd6592c37bb2:42639 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T22:14:41,802 INFO [RS:1;bd6592c37bb2:38125 {}] regionserver.HRegionServer(3091): Received CLOSE for 79d315a497ec3ad891988cb800bae337 2024-11-24T22:14:41,802 INFO [RS:2;bd6592c37bb2:42639 {}] regionserver.HRegionServer(3091): Received CLOSE for 5160ff1dd0e56b08fbca2ffe581141d4 2024-11-24T22:14:41,802 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38125-0x1014c0b79ef0002, quorum=127.0.0.1:56583, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T22:14:41,802 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36279-0x1014c0b79ef0000, quorum=127.0.0.1:56583, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T22:14:41,802 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T22:14:41,802 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T22:14:41,805 INFO [RS:1;bd6592c37bb2:38125 {}] regionserver.HRegionServer(3091): Received CLOSE for 84dd509902edc43083bc78fe4b70922c 2024-11-24T22:14:41,805 INFO [RS:1;bd6592c37bb2:38125 {}] regionserver.HRegionServer(3091): Received CLOSE for 4279880609a62ccb9e9fb29e79bdf7e7 2024-11-24T22:14:41,805 INFO [RS:1;bd6592c37bb2:38125 {}] regionserver.HRegionServer(3091): Received CLOSE for 967acf99df098c28d8f837a9f0746a90 2024-11-24T22:14:41,805 INFO [RS:1;bd6592c37bb2:38125 {}] regionserver.HRegionServer(3091): Received CLOSE for ce1f0bcb3026cb1cbbd9e63c21b83f99 2024-11-24T22:14:41,805 INFO [RS:1;bd6592c37bb2:38125 {}] regionserver.HRegionServer(3091): Received CLOSE for 9b2e9ce57ffd0cd61c72602e5f6d9cae 2024-11-24T22:14:41,805 INFO [RS:1;bd6592c37bb2:38125 {}] regionserver.HRegionServer(3091): Received CLOSE for 94f2fa48145fe3bdeffbfded6a4edd73 2024-11-24T22:14:41,805 INFO [RS:1;bd6592c37bb2:38125 {}] regionserver.HRegionServer(3091): Received CLOSE for 516ead6fe1773c4da8708cdc5713f526 2024-11-24T22:14:41,805 INFO [RS:1;bd6592c37bb2:38125 {}] regionserver.HRegionServer(3091): Received CLOSE for 736f44b8f348e2a5a96dfd3abc5e62b2 2024-11-24T22:14:41,805 INFO [RS:1;bd6592c37bb2:38125 {}] regionserver.HRegionServer(3091): Received CLOSE for cca6a66154e5c299fad3a247afbfb5fd 2024-11-24T22:14:41,805 INFO [RS:1;bd6592c37bb2:38125 {}] regionserver.HRegionServer(3091): Received CLOSE for 79c2dd6f654fabefe0fc06b8199e8562 2024-11-24T22:14:41,805 INFO [RS:1;bd6592c37bb2:38125 {}] regionserver.HRegionServer(3091): Received CLOSE for acdab7e426c93b10b34b4d331b21cef0 2024-11-24T22:14:41,805 INFO [RS:1;bd6592c37bb2:38125 {}] regionserver.HRegionServer(959): stopping server bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:41,805 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 79d315a497ec3ad891988cb800bae337, disabling compactions & flushes 2024-11-24T22:14:41,805 INFO [RS:1;bd6592c37bb2:38125 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T22:14:41,805 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,yyy,1732486472486.79d315a497ec3ad891988cb800bae337. 2024-11-24T22:14:41,805 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1732486472486.79d315a497ec3ad891988cb800bae337. 2024-11-24T22:14:41,805 INFO [RS:1;bd6592c37bb2:38125 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;bd6592c37bb2:38125. 2024-11-24T22:14:41,805 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1732486472486.79d315a497ec3ad891988cb800bae337. after waiting 0 ms 2024-11-24T22:14:41,805 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1732486472486.79d315a497ec3ad891988cb800bae337. 2024-11-24T22:14:41,805 DEBUG [RS:1;bd6592c37bb2:38125 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T22:14:41,805 DEBUG [RS:1;bd6592c37bb2:38125 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:41,805 INFO [RS:1;bd6592c37bb2:38125 {}] regionserver.HRegionServer(1321): Waiting on 12 regions to close 2024-11-24T22:14:41,806 DEBUG [RS:1;bd6592c37bb2:38125 {}] regionserver.HRegionServer(1351): Waiting on 4279880609a62ccb9e9fb29e79bdf7e7, 516ead6fe1773c4da8708cdc5713f526, 698dd9d64bdaf6a81e797d2590ff86f9, 736f44b8f348e2a5a96dfd3abc5e62b2, 79c2dd6f654fabefe0fc06b8199e8562, 79d315a497ec3ad891988cb800bae337, 84dd509902edc43083bc78fe4b70922c, 94f2fa48145fe3bdeffbfded6a4edd73, 967acf99df098c28d8f837a9f0746a90, 9b2e9ce57ffd0cd61c72602e5f6d9cae, acdab7e426c93b10b34b4d331b21cef0, cca6a66154e5c299fad3a247afbfb5fd, ce1f0bcb3026cb1cbbd9e63c21b83f99 2024-11-24T22:14:41,806 INFO [RS:2;bd6592c37bb2:42639 {}] regionserver.HRegionServer(3091): Received CLOSE for 46b79f5a3db63445091682b5e72f635a 2024-11-24T22:14:41,806 INFO [RS:2;bd6592c37bb2:42639 {}] regionserver.HRegionServer(3091): Received CLOSE for c802d82f160c5269119d007388afb85e 2024-11-24T22:14:41,806 INFO [RS:2;bd6592c37bb2:42639 {}] regionserver.HRegionServer(3091): Received CLOSE for f72c935213180e3ff989af795022b7e4 2024-11-24T22:14:41,806 INFO [RS:2;bd6592c37bb2:42639 {}] regionserver.HRegionServer(3091): Received CLOSE for 72a41015f7b88194cc3a08feb66aa4f9 2024-11-24T22:14:41,806 INFO [RS:2;bd6592c37bb2:42639 {}] regionserver.HRegionServer(3091): Received CLOSE for 0b86957c7b81466a818bbff45fac751f 2024-11-24T22:14:41,806 INFO [RS:2;bd6592c37bb2:42639 {}] regionserver.HRegionServer(3091): Received CLOSE for 83e58b0e0989e11c83283bf7bd41306c 2024-11-24T22:14:41,806 INFO [RS:2;bd6592c37bb2:42639 {}] regionserver.HRegionServer(3091): Received CLOSE for c68a44a11eeaccf8a3c2f49ba1446c1e 2024-11-24T22:14:41,806 INFO [RS:2;bd6592c37bb2:42639 {}] regionserver.HRegionServer(3091): Received CLOSE for ed5b70c013e388180062b1513b980a07 2024-11-24T22:14:41,806 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5160ff1dd0e56b08fbca2ffe581141d4, disabling compactions & flushes 2024-11-24T22:14:41,806 INFO [RS:2;bd6592c37bb2:42639 {}] regionserver.HRegionServer(3091): Received CLOSE for 50d9b38af2d6413e500860133d150aa8 2024-11-24T22:14:41,806 INFO [RS:2;bd6592c37bb2:42639 {}] regionserver.HRegionServer(3091): Received CLOSE for dc147c653b22684b8e41092cd6a0c8eb 2024-11-24T22:14:41,806 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,nnn,1732486472486.5160ff1dd0e56b08fbca2ffe581141d4. 2024-11-24T22:14:41,806 INFO [RS:2;bd6592c37bb2:42639 {}] regionserver.HRegionServer(3091): Received CLOSE for fc9f96fd4c44240a85e6b2c001e46153 2024-11-24T22:14:41,806 INFO [RS:2;bd6592c37bb2:42639 {}] regionserver.HRegionServer(959): stopping server bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:41,806 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1732486472486.5160ff1dd0e56b08fbca2ffe581141d4. 2024-11-24T22:14:41,806 INFO [RS:2;bd6592c37bb2:42639 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T22:14:41,806 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1732486472486.5160ff1dd0e56b08fbca2ffe581141d4. after waiting 0 ms 2024-11-24T22:14:41,806 INFO [RS:2;bd6592c37bb2:42639 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;bd6592c37bb2:42639. 2024-11-24T22:14:41,806 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1732486472486.5160ff1dd0e56b08fbca2ffe581141d4. 2024-11-24T22:14:41,806 DEBUG [RS:2;bd6592c37bb2:42639 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T22:14:41,806 DEBUG [RS:2;bd6592c37bb2:42639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:41,807 INFO [RS:2;bd6592c37bb2:42639 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T22:14:41,807 INFO [RS:2;bd6592c37bb2:42639 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T22:14:41,807 INFO [RS:2;bd6592c37bb2:42639 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T22:14:41,807 INFO [RS:2;bd6592c37bb2:42639 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-24T22:14:41,807 INFO [RS:2;bd6592c37bb2:42639 {}] regionserver.HRegionServer(1321): Waiting on 13 regions to close 2024-11-24T22:14:41,807 DEBUG [RS:2;bd6592c37bb2:42639 {}] regionserver.HRegionServer(1351): Waiting on 0b86957c7b81466a818bbff45fac751f, 1588230740, 46b79f5a3db63445091682b5e72f635a, 4e91326d9778777879bc895ecb241af8, 50d9b38af2d6413e500860133d150aa8, 5160ff1dd0e56b08fbca2ffe581141d4, 72a41015f7b88194cc3a08feb66aa4f9, 83e58b0e0989e11c83283bf7bd41306c, c68a44a11eeaccf8a3c2f49ba1446c1e, c802d82f160c5269119d007388afb85e, dc147c653b22684b8e41092cd6a0c8eb, ed5b70c013e388180062b1513b980a07, f72c935213180e3ff989af795022b7e4, fc9f96fd4c44240a85e6b2c001e46153 2024-11-24T22:14:41,808 DEBUG [RS_CLOSE_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T22:14:41,808 INFO [RS_CLOSE_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T22:14:41,808 DEBUG [RS_CLOSE_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T22:14:41,808 DEBUG [RS_CLOSE_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T22:14:41,808 DEBUG [RS_CLOSE_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T22:14:41,808 INFO [RS_CLOSE_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.64 KB heapSize=5.42 KB 2024-11-24T22:14:41,817 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/79d315a497ec3ad891988cb800bae337/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T22:14:41,817 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/5160ff1dd0e56b08fbca2ffe581141d4/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T22:14:41,817 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,yyy,1732486472486.79d315a497ec3ad891988cb800bae337. 2024-11-24T22:14:41,818 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 79d315a497ec3ad891988cb800bae337: Waiting for close lock at 1732486481805Running coprocessor pre-close hooks at 1732486481805Disabling compacts and flushes for region at 1732486481805Disabling writes for close at 1732486481805Writing region close event to WAL at 1732486481807 (+2 ms)Running coprocessor post-close hooks at 1732486481817 (+10 ms)Closed at 1732486481817 2024-11-24T22:14:41,818 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,yyy,1732486472486.79d315a497ec3ad891988cb800bae337. 2024-11-24T22:14:41,818 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 84dd509902edc43083bc78fe4b70922c, disabling compactions & flushes 2024-11-24T22:14:41,818 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,sss,1732486472486.84dd509902edc43083bc78fe4b70922c. 2024-11-24T22:14:41,818 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1732486472486.84dd509902edc43083bc78fe4b70922c. 2024-11-24T22:14:41,818 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,sss,1732486472486.84dd509902edc43083bc78fe4b70922c. after waiting 0 ms 2024-11-24T22:14:41,818 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,sss,1732486472486.84dd509902edc43083bc78fe4b70922c. 2024-11-24T22:14:41,819 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,nnn,1732486472486.5160ff1dd0e56b08fbca2ffe581141d4. 2024-11-24T22:14:41,819 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5160ff1dd0e56b08fbca2ffe581141d4: Waiting for close lock at 1732486481806Running coprocessor pre-close hooks at 1732486481806Disabling compacts and flushes for region at 1732486481806Disabling writes for close at 1732486481806Writing region close event to WAL at 1732486481807 (+1 ms)Running coprocessor post-close hooks at 1732486481818 (+11 ms)Closed at 1732486481819 (+1 ms) 2024-11-24T22:14:41,819 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,nnn,1732486472486.5160ff1dd0e56b08fbca2ffe581141d4. 2024-11-24T22:14:41,819 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 46b79f5a3db63445091682b5e72f635a, disabling compactions & flushes 2024-11-24T22:14:41,819 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,eee,1732486472486.46b79f5a3db63445091682b5e72f635a. 2024-11-24T22:14:41,819 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1732486472486.46b79f5a3db63445091682b5e72f635a. 2024-11-24T22:14:41,819 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,eee,1732486472486.46b79f5a3db63445091682b5e72f635a. after waiting 0 ms 2024-11-24T22:14:41,820 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,eee,1732486472486.46b79f5a3db63445091682b5e72f635a. 2024-11-24T22:14:41,822 DEBUG [RS_CLOSE_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/hbase/meta/1588230740/.tmp/info/8b2f76479c26407ba2430badc68d131b is 138, key is test,ddd,1732486472486.698dd9d64bdaf6a81e797d2590ff86f9./info:regioninfo/1732486478393/Put/seqid=0 2024-11-24T22:14:41,823 INFO [regionserver/bd6592c37bb2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T22:14:41,824 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/84dd509902edc43083bc78fe4b70922c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T22:14:41,825 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,sss,1732486472486.84dd509902edc43083bc78fe4b70922c. 2024-11-24T22:14:41,825 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 84dd509902edc43083bc78fe4b70922c: Waiting for close lock at 1732486481818Running coprocessor pre-close hooks at 1732486481818Disabling compacts and flushes for region at 1732486481818Disabling writes for close at 1732486481818Writing region close event to WAL at 1732486481819 (+1 ms)Running coprocessor post-close hooks at 1732486481825 (+6 ms)Closed at 1732486481825 2024-11-24T22:14:41,825 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,sss,1732486472486.84dd509902edc43083bc78fe4b70922c. 2024-11-24T22:14:41,825 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4279880609a62ccb9e9fb29e79bdf7e7, disabling compactions & flushes 2024-11-24T22:14:41,825 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,bbb,1732486472486.4279880609a62ccb9e9fb29e79bdf7e7. 2024-11-24T22:14:41,825 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1732486472486.4279880609a62ccb9e9fb29e79bdf7e7. 2024-11-24T22:14:41,825 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1732486472486.4279880609a62ccb9e9fb29e79bdf7e7. after waiting 0 ms 2024-11-24T22:14:41,825 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1732486472486.4279880609a62ccb9e9fb29e79bdf7e7. 2024-11-24T22:14:41,831 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/4279880609a62ccb9e9fb29e79bdf7e7/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T22:14:41,831 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,bbb,1732486472486.4279880609a62ccb9e9fb29e79bdf7e7. 2024-11-24T22:14:41,831 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4279880609a62ccb9e9fb29e79bdf7e7: Waiting for close lock at 1732486481825Running coprocessor pre-close hooks at 1732486481825Disabling compacts and flushes for region at 1732486481825Disabling writes for close at 1732486481825Writing region close event to WAL at 1732486481826 (+1 ms)Running coprocessor post-close hooks at 1732486481831 (+5 ms)Closed at 1732486481831 2024-11-24T22:14:41,832 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,bbb,1732486472486.4279880609a62ccb9e9fb29e79bdf7e7. 2024-11-24T22:14:41,832 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 967acf99df098c28d8f837a9f0746a90, disabling compactions & flushes 2024-11-24T22:14:41,832 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,lll,1732486472486.967acf99df098c28d8f837a9f0746a90. 2024-11-24T22:14:41,832 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1732486472486.967acf99df098c28d8f837a9f0746a90. 2024-11-24T22:14:41,832 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,lll,1732486472486.967acf99df098c28d8f837a9f0746a90. after waiting 0 ms 2024-11-24T22:14:41,832 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,lll,1732486472486.967acf99df098c28d8f837a9f0746a90. 2024-11-24T22:14:41,837 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/46b79f5a3db63445091682b5e72f635a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T22:14:41,837 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/967acf99df098c28d8f837a9f0746a90/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T22:14:41,838 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,eee,1732486472486.46b79f5a3db63445091682b5e72f635a. 2024-11-24T22:14:41,838 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 46b79f5a3db63445091682b5e72f635a: Waiting for close lock at 1732486481819Running coprocessor pre-close hooks at 1732486481819Disabling compacts and flushes for region at 1732486481819Disabling writes for close at 1732486481819Writing region close event to WAL at 1732486481824 (+5 ms)Running coprocessor post-close hooks at 1732486481838 (+14 ms)Closed at 1732486481838 2024-11-24T22:14:41,838 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,eee,1732486472486.46b79f5a3db63445091682b5e72f635a. 2024-11-24T22:14:41,838 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c802d82f160c5269119d007388afb85e, disabling compactions & flushes 2024-11-24T22:14:41,838 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,rrr,1732486472486.c802d82f160c5269119d007388afb85e. 2024-11-24T22:14:41,838 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1732486472486.c802d82f160c5269119d007388afb85e. 2024-11-24T22:14:41,838 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1732486472486.c802d82f160c5269119d007388afb85e. after waiting 0 ms 2024-11-24T22:14:41,838 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1732486472486.c802d82f160c5269119d007388afb85e. 2024-11-24T22:14:41,839 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,lll,1732486472486.967acf99df098c28d8f837a9f0746a90. 2024-11-24T22:14:41,839 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 967acf99df098c28d8f837a9f0746a90: Waiting for close lock at 1732486481832Running coprocessor pre-close hooks at 1732486481832Disabling compacts and flushes for region at 1732486481832Disabling writes for close at 1732486481832Writing region close event to WAL at 1732486481832Running coprocessor post-close hooks at 1732486481839 (+7 ms)Closed at 1732486481839 2024-11-24T22:14:41,839 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,lll,1732486472486.967acf99df098c28d8f837a9f0746a90. 2024-11-24T22:14:41,839 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ce1f0bcb3026cb1cbbd9e63c21b83f99, disabling compactions & flushes 2024-11-24T22:14:41,839 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ooo,1732486472486.ce1f0bcb3026cb1cbbd9e63c21b83f99. 2024-11-24T22:14:41,839 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1732486472486.ce1f0bcb3026cb1cbbd9e63c21b83f99. 2024-11-24T22:14:41,839 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1732486472486.ce1f0bcb3026cb1cbbd9e63c21b83f99. after waiting 0 ms 2024-11-24T22:14:41,839 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1732486472486.ce1f0bcb3026cb1cbbd9e63c21b83f99. 2024-11-24T22:14:41,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35383 is added to blk_1073741869_1045 (size=8177) 2024-11-24T22:14:41,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741869_1045 (size=8177) 2024-11-24T22:14:41,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34013 is added to blk_1073741869_1045 (size=8177) 2024-11-24T22:14:41,841 INFO [RS_CLOSE_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.64 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/hbase/meta/1588230740/.tmp/info/8b2f76479c26407ba2430badc68d131b 2024-11-24T22:14:41,849 WARN [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,,1732486472486.4e91326d9778777879bc895ecb241af8., aborting... java.io.IOException: Failed to report opened region to master: test,,1732486472486.4e91326d9778777879bc895ecb241af8. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:41,849 ERROR [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2440): ***** ABORTING region server bd6592c37bb2,42639,1732486469476: Failed to open region test,,1732486472486.4e91326d9778777879bc895ecb241af8. and can not recover ***** java.io.IOException: Failed to report opened region to master: test,,1732486472486.4e91326d9778777879bc895ecb241af8. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:41,849 ERROR [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-24T22:14:41,849 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-24T22:14:41,850 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-24T22:14:41,852 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-24T22:14:41,853 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-24T22:14:41,855 WARN [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,ddd,1732486472486.698dd9d64bdaf6a81e797d2590ff86f9., aborting... java.io.IOException: Failed to report opened region to master: test,ddd,1732486472486.698dd9d64bdaf6a81e797d2590ff86f9. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:41,856 ERROR [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegionServer(2440): ***** ABORTING region server bd6592c37bb2,38125,1732486469405: Failed to open region test,ddd,1732486472486.698dd9d64bdaf6a81e797d2590ff86f9. and can not recover ***** java.io.IOException: Failed to report opened region to master: test,ddd,1732486472486.698dd9d64bdaf6a81e797d2590ff86f9. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T22:14:41,856 ERROR [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-24T22:14:41,856 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-24T22:14:41,856 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-24T22:14:41,859 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-24T22:14:41,859 DEBUG [RS_CLOSE_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/hbase/meta/1588230740/.tmp/info/8b2f76479c26407ba2430badc68d131b as hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/hbase/meta/1588230740/info/8b2f76479c26407ba2430badc68d131b 2024-11-24T22:14:41,860 DEBUG [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-24T22:14:41,865 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 510652672 }, "NonHeapMemoryUsage": { "committed": 164364288, "init": 7667712, "max": -1, "used": 161695728 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "bd6592c37bb2", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 35970, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 107, "ProcessCallTime_min": 0, "ProcessCallTime_max": 32, "ProcessCallTime_mean": 3, "ProcessCallTime_25th_percentile": 8, "ProcessCallTime_median": 16, "ProcessCallTime_75th_percentile": 24, "ProcessCallTime_90th_percentile": 28, "ProcessCallTime_95th_percentile": 30, "ProcessCallTime_98th_percentile": 31, "ProcessCallTime_99th_percentile": 31, "ProcessCallTime_99.9th_percentile": 31, "ProcessCallTime_TimeRangeCount_0-1": 107, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 107, "QueueCallTime_min": 0, "QueueCallTime_max": 71, "QueueCallTime_mean": 7, "QueueCallTime_25th_percentile": 17, "QueueCallTime_median": 35, "QueueCallTime_75th_percentile": 53, "QueueCallTime_90th_percentile": 63, "QueueCallTime_95th_percentile": 67, "QueueCallTime_98th_percentile": 69, "QueueCallTime_99th_percentile": 70, "QueueCallTime_99.9th_percentile": 70, "QueueCallTime_TimeRangeCount_0-1": 107, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 107, "TotalCallTime_min": 0, "TotalCallTime_max": 73, "TotalCallTime_mean": 11, "TotalCallTime_25th_percentile": 18, "TotalCallTime_median": 36, "TotalCallTime_75th_percentile": 54, "TotalCallTime_90th_percentile": 65, "TotalCallTime_95th_percentile": 69, "TotalCallTime_98th_percentile": 71, "TotalCallTime_99th_percentile": 72, "TotalCallTime_99.9th_percentile": 72, "TotalCallTime_TimeRangeCount_0-1": 107, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 107, "ResponseSize_min": 0, "ResponseSize_max": 22359, "ResponseSize_mean": 639, "ResponseSize_25th_percentile": 5589, "ResponseSize_median": 11179, "ResponseSize_75th_percentile": 16769, "ResponseSize_90th_percentile": 20123, "ResponseSize_95th_percentile": 21241, "ResponseSize_98th_percentile": 21911, "ResponseSize_99th_percentile": 22135, "ResponseSize_99.9th_percentile": 22336, "ResponseSize_SizeRangeCount_0-10": 107, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 107, "RequestSize_min": 81, "RequestSize_max": 8095, "RequestSize_mean": 305, "RequestSize_25th_percentile": 2084, "RequestSize_median": 4088, "RequestSize_75th_percentile": 6091, "RequestSize_90th_percentile": 7293, "RequestSize_95th_percentile": 7694, "RequestSize_98th_percentile": 7934, "RequestSize_99th_percentile": 8014, "RequestSize_99.9th_percentile": 8086, "RequestSize_SizeRangeCount_0-10": 107, "sentBytes": 54813 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "bd6592c37bb2", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:56583", "tag.serverName": "bd6592c37bb2,42639,1732486469476", "tag.clusterId": "905a4a1c-dd04-42c5-bf3f-b8b8bda6274a", "tag.Context": "regionserver", "tag.Hostname": "bd6592c37bb2", "regionCount": 9, "storeCount": 9, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 2304, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1732486469476, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 0, "localBytesRead": 0, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 1.8, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 1605, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 9, "totalRowActionRequestCount": 9, "readRequestCount": 9, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 9, "rpcScanRequestCount": 9, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 9, "ScanTime_min": 0, "ScanTime_max": 1, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "ScanTime_TimeRangeCount_0-1": 9, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 9, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "ScanSize_SizeRangeCount_0-10": 9, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-24T22:14:41,865 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/ce1f0bcb3026cb1cbbd9e63c21b83f99/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T22:14:41,866 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ooo,1732486472486.ce1f0bcb3026cb1cbbd9e63c21b83f99. 2024-11-24T22:14:41,866 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ce1f0bcb3026cb1cbbd9e63c21b83f99: Waiting for close lock at 1732486481839Running coprocessor pre-close hooks at 1732486481839Disabling compacts and flushes for region at 1732486481839Disabling writes for close at 1732486481839Writing region close event to WAL at 1732486481848 (+9 ms)Running coprocessor post-close hooks at 1732486481866 (+18 ms)Closed at 1732486481866 2024-11-24T22:14:41,866 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ooo,1732486472486.ce1f0bcb3026cb1cbbd9e63c21b83f99. 2024-11-24T22:14:41,866 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9b2e9ce57ffd0cd61c72602e5f6d9cae, disabling compactions & flushes 2024-11-24T22:14:41,866 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,jjj,1732486472486.9b2e9ce57ffd0cd61c72602e5f6d9cae. 2024-11-24T22:14:41,866 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1732486472486.9b2e9ce57ffd0cd61c72602e5f6d9cae. 2024-11-24T22:14:41,866 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1732486472486.9b2e9ce57ffd0cd61c72602e5f6d9cae. after waiting 0 ms 2024-11-24T22:14:41,866 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1732486472486.9b2e9ce57ffd0cd61c72602e5f6d9cae. 2024-11-24T22:14:41,867 WARN [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2470): Unable to report fatal error to master org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=bd6592c37bb2:36279 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:344) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportRSFatalError(RegionServerStatusProtos.java:17290) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.abort(HRegionServer.java:2467) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abortRegionServer(SingleProcessHBaseCluster.java:192) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$2.run(SingleProcessHBaseCluster.java:185) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abort(SingleProcessHBaseCluster.java:182) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.handleException(AssignRegionHandler.java:173) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:106) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=bd6592c37bb2:36279 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 15 more Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 15 more 2024-11-24T22:14:41,870 INFO [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 510652672 }, "NonHeapMemoryUsage": { "committed": 164364288, "init": 7667712, "max": -1, "used": 161719640 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "bd6592c37bb2", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 35970, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 107, "ProcessCallTime_min": 0, "ProcessCallTime_max": 32, "ProcessCallTime_mean": 3, "ProcessCallTime_25th_percentile": 8, "ProcessCallTime_median": 16, "ProcessCallTime_75th_percentile": 24, "ProcessCallTime_90th_percentile": 28, "ProcessCallTime_95th_percentile": 30, "ProcessCallTime_98th_percentile": 31, "ProcessCallTime_99th_percentile": 31, "ProcessCallTime_99.9th_percentile": 31, "ProcessCallTime_TimeRangeCount_0-1": 107, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 107, "QueueCallTime_min": 0, "QueueCallTime_max": 71, "QueueCallTime_mean": 7, "QueueCallTime_25th_percentile": 17, "QueueCallTime_median": 35, "QueueCallTime_75th_percentile": 53, "QueueCallTime_90th_percentile": 63, "QueueCallTime_95th_percentile": 67, "QueueCallTime_98th_percentile": 69, "QueueCallTime_99th_percentile": 70, "QueueCallTime_99.9th_percentile": 70, "QueueCallTime_TimeRangeCount_0-1": 107, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 107, "TotalCallTime_min": 0, "TotalCallTime_max": 73, "TotalCallTime_mean": 11, "TotalCallTime_25th_percentile": 18, "TotalCallTime_median": 36, "TotalCallTime_75th_percentile": 54, "TotalCallTime_90th_percentile": 65, "TotalCallTime_95th_percentile": 69, "TotalCallTime_98th_percentile": 71, "TotalCallTime_99th_percentile": 72, "TotalCallTime_99.9th_percentile": 72, "TotalCallTime_TimeRangeCount_0-1": 107, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 107, "ResponseSize_min": 0, "ResponseSize_max": 22359, "ResponseSize_mean": 639, "ResponseSize_25th_percentile": 5589, "ResponseSize_median": 11179, "ResponseSize_75th_percentile": 16769, "ResponseSize_90th_percentile": 20123, "ResponseSize_95th_percentile": 21241, "ResponseSize_98th_percentile": 21911, "ResponseSize_99th_percentile": 22135, "ResponseSize_99.9th_percentile": 22336, "ResponseSize_SizeRangeCount_0-10": 107, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 107, "RequestSize_min": 81, "RequestSize_max": 8095, "RequestSize_mean": 305, "RequestSize_25th_percentile": 2084, "RequestSize_median": 4088, "RequestSize_75th_percentile": 6091, "RequestSize_90th_percentile": 7293, "RequestSize_95th_percentile": 7694, "RequestSize_98th_percentile": 7934, "RequestSize_99th_percentile": 8014, "RequestSize_99.9th_percentile": 8086, "RequestSize_SizeRangeCount_0-10": 107, "sentBytes": 54813 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "bd6592c37bb2", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:56583", "tag.serverName": "bd6592c37bb2,42639,1732486469476", "tag.clusterId": "905a4a1c-dd04-42c5-bf3f-b8b8bda6274a", "tag.Context": "regionserver", "tag.Hostname": "bd6592c37bb2", "regionCount": 9, "storeCount": 9, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 2304, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1732486469476, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 0, "localBytesRead": 0, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 1.8, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 1605, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 9, "totalRowActionRequestCount": 9, "readRequestCount": 9, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 9, "rpcScanRequestCount": 9, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 9, "ScanTime_min": 0, "ScanTime_max": 1, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "ScanTime_TimeRangeCount_0-1": 9, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 9, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "ScanSize_SizeRangeCount_0-10": 9, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-24T22:14:41,871 WARN [RS_OPEN_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegionServer(2470): Unable to report fatal error to master org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=bd6592c37bb2:36279 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:344) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportRSFatalError(RegionServerStatusProtos.java:17290) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.abort(HRegionServer.java:2467) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abortRegionServer(SingleProcessHBaseCluster.java:192) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$2.run(SingleProcessHBaseCluster.java:185) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abort(SingleProcessHBaseCluster.java:182) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.handleException(AssignRegionHandler.java:173) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:106) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=bd6592c37bb2:36279 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 15 more Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 15 more 2024-11-24T22:14:41,879 INFO [RS_CLOSE_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/hbase/meta/1588230740/info/8b2f76479c26407ba2430badc68d131b, entries=24, sequenceid=75, filesize=8.0 K 2024-11-24T22:14:41,880 INFO [RS_CLOSE_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.64 KB/2708, heapSize ~4.66 KB/4768, currentSize=0 B/0 for 1588230740 in 72ms, sequenceid=75, compaction requested=false 2024-11-24T22:14:41,886 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/c802d82f160c5269119d007388afb85e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T22:14:41,887 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,rrr,1732486472486.c802d82f160c5269119d007388afb85e. 2024-11-24T22:14:41,887 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c802d82f160c5269119d007388afb85e: Waiting for close lock at 1732486481838Running coprocessor pre-close hooks at 1732486481838Disabling compacts and flushes for region at 1732486481838Disabling writes for close at 1732486481838Writing region close event to WAL at 1732486481841 (+3 ms)Running coprocessor post-close hooks at 1732486481887 (+46 ms)Closed at 1732486481887 2024-11-24T22:14:41,887 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,rrr,1732486472486.c802d82f160c5269119d007388afb85e. 2024-11-24T22:14:41,887 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f72c935213180e3ff989af795022b7e4, disabling compactions & flushes 2024-11-24T22:14:41,887 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ggg,1732486472486.f72c935213180e3ff989af795022b7e4. 2024-11-24T22:14:41,887 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1732486472486.f72c935213180e3ff989af795022b7e4. 2024-11-24T22:14:41,887 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1732486472486.f72c935213180e3ff989af795022b7e4. after waiting 0 ms 2024-11-24T22:14:41,887 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1732486472486.f72c935213180e3ff989af795022b7e4. 2024-11-24T22:14:41,888 INFO [regionserver/bd6592c37bb2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T22:14:41,893 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/9b2e9ce57ffd0cd61c72602e5f6d9cae/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T22:14:41,894 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,jjj,1732486472486.9b2e9ce57ffd0cd61c72602e5f6d9cae. 2024-11-24T22:14:41,894 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9b2e9ce57ffd0cd61c72602e5f6d9cae: Waiting for close lock at 1732486481866Running coprocessor pre-close hooks at 1732486481866Disabling compacts and flushes for region at 1732486481866Disabling writes for close at 1732486481866Writing region close event to WAL at 1732486481878 (+12 ms)Running coprocessor post-close hooks at 1732486481894 (+16 ms)Closed at 1732486481894 2024-11-24T22:14:41,894 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,jjj,1732486472486.9b2e9ce57ffd0cd61c72602e5f6d9cae. 2024-11-24T22:14:41,894 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 94f2fa48145fe3bdeffbfded6a4edd73, disabling compactions & flushes 2024-11-24T22:14:41,894 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,iii,1732486472486.94f2fa48145fe3bdeffbfded6a4edd73. 2024-11-24T22:14:41,894 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1732486472486.94f2fa48145fe3bdeffbfded6a4edd73. 2024-11-24T22:14:41,894 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,iii,1732486472486.94f2fa48145fe3bdeffbfded6a4edd73. after waiting 0 ms 2024-11-24T22:14:41,894 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,iii,1732486472486.94f2fa48145fe3bdeffbfded6a4edd73. 2024-11-24T22:14:41,906 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/f72c935213180e3ff989af795022b7e4/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T22:14:41,907 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ggg,1732486472486.f72c935213180e3ff989af795022b7e4. 2024-11-24T22:14:41,907 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f72c935213180e3ff989af795022b7e4: Waiting for close lock at 1732486481887Running coprocessor pre-close hooks at 1732486481887Disabling compacts and flushes for region at 1732486481887Disabling writes for close at 1732486481887Writing region close event to WAL at 1732486481896 (+9 ms)Running coprocessor post-close hooks at 1732486481907 (+11 ms)Closed at 1732486481907 2024-11-24T22:14:41,907 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ggg,1732486472486.f72c935213180e3ff989af795022b7e4. 2024-11-24T22:14:41,907 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 72a41015f7b88194cc3a08feb66aa4f9, disabling compactions & flushes 2024-11-24T22:14:41,907 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,hhh,1732486472486.72a41015f7b88194cc3a08feb66aa4f9. 2024-11-24T22:14:41,907 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1732486472486.72a41015f7b88194cc3a08feb66aa4f9. 2024-11-24T22:14:41,907 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1732486472486.72a41015f7b88194cc3a08feb66aa4f9. after waiting 0 ms 2024-11-24T22:14:41,907 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1732486472486.72a41015f7b88194cc3a08feb66aa4f9. 2024-11-24T22:14:41,911 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/94f2fa48145fe3bdeffbfded6a4edd73/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T22:14:41,912 DEBUG [RS_CLOSE_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/hbase/meta/1588230740/recovered.edits/78.seqid, newMaxSeqId=78, maxSeqId=64 2024-11-24T22:14:41,913 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,iii,1732486472486.94f2fa48145fe3bdeffbfded6a4edd73. 2024-11-24T22:14:41,913 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 94f2fa48145fe3bdeffbfded6a4edd73: Waiting for close lock at 1732486481894Running coprocessor pre-close hooks at 1732486481894Disabling compacts and flushes for region at 1732486481894Disabling writes for close at 1732486481894Writing region close event to WAL at 1732486481905 (+11 ms)Running coprocessor post-close hooks at 1732486481913 (+8 ms)Closed at 1732486481913 2024-11-24T22:14:41,913 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,iii,1732486472486.94f2fa48145fe3bdeffbfded6a4edd73. 2024-11-24T22:14:41,913 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 516ead6fe1773c4da8708cdc5713f526, disabling compactions & flushes 2024-11-24T22:14:41,913 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,xxx,1732486472486.516ead6fe1773c4da8708cdc5713f526. 2024-11-24T22:14:41,913 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1732486472486.516ead6fe1773c4da8708cdc5713f526. 2024-11-24T22:14:41,913 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1732486472486.516ead6fe1773c4da8708cdc5713f526. after waiting 0 ms 2024-11-24T22:14:41,913 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1732486472486.516ead6fe1773c4da8708cdc5713f526. 2024-11-24T22:14:41,917 DEBUG [RS_CLOSE_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T22:14:41,917 INFO [RS_CLOSE_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T22:14:41,917 DEBUG [RS_CLOSE_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732486481808Running coprocessor pre-close hooks at 1732486481808Disabling compacts and flushes for region at 1732486481808Disabling writes for close at 1732486481808Obtaining lock to block concurrent updates at 1732486481808Preparing flush snapshotting stores in 1588230740 at 1732486481808Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=2708, getHeapSize=5488, getOffHeapSize=0, getCellsCount=24 at 1732486481815 (+7 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732486481816 (+1 ms)Flushing 1588230740/info: creating writer at 1732486481817 (+1 ms)Flushing 1588230740/info: appending metadata at 1732486481822 (+5 ms)Flushing 1588230740/info: closing flushed file at 1732486481822Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7e6918de: reopening flushed file at 1732486481856 (+34 ms)Finished flush of dataSize ~2.64 KB/2708, heapSize ~4.66 KB/4768, currentSize=0 B/0 for 1588230740 in 72ms, sequenceid=75, compaction requested=false at 1732486481880 (+24 ms)Writing region close event to WAL at 1732486481906 (+26 ms)Running coprocessor post-close hooks at 1732486481917 (+11 ms)Closed at 1732486481917 2024-11-24T22:14:41,917 DEBUG [RS_CLOSE_META-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-24T22:14:41,925 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/516ead6fe1773c4da8708cdc5713f526/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T22:14:41,926 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,xxx,1732486472486.516ead6fe1773c4da8708cdc5713f526. 2024-11-24T22:14:41,926 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 516ead6fe1773c4da8708cdc5713f526: Waiting for close lock at 1732486481913Running coprocessor pre-close hooks at 1732486481913Disabling compacts and flushes for region at 1732486481913Disabling writes for close at 1732486481913Writing region close event to WAL at 1732486481914 (+1 ms)Running coprocessor post-close hooks at 1732486481926 (+12 ms)Closed at 1732486481926 2024-11-24T22:14:41,926 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,xxx,1732486472486.516ead6fe1773c4da8708cdc5713f526. 2024-11-24T22:14:41,926 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 736f44b8f348e2a5a96dfd3abc5e62b2, disabling compactions & flushes 2024-11-24T22:14:41,926 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,fff,1732486472486.736f44b8f348e2a5a96dfd3abc5e62b2. 2024-11-24T22:14:41,926 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1732486472486.736f44b8f348e2a5a96dfd3abc5e62b2. 2024-11-24T22:14:41,926 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,fff,1732486472486.736f44b8f348e2a5a96dfd3abc5e62b2. after waiting 0 ms 2024-11-24T22:14:41,926 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,fff,1732486472486.736f44b8f348e2a5a96dfd3abc5e62b2. 2024-11-24T22:14:41,926 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/72a41015f7b88194cc3a08feb66aa4f9/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T22:14:41,927 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,hhh,1732486472486.72a41015f7b88194cc3a08feb66aa4f9. 2024-11-24T22:14:41,927 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 72a41015f7b88194cc3a08feb66aa4f9: Waiting for close lock at 1732486481907Running coprocessor pre-close hooks at 1732486481907Disabling compacts and flushes for region at 1732486481907Disabling writes for close at 1732486481907Writing region close event to WAL at 1732486481908 (+1 ms)Running coprocessor post-close hooks at 1732486481927 (+19 ms)Closed at 1732486481927 2024-11-24T22:14:41,927 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,hhh,1732486472486.72a41015f7b88194cc3a08feb66aa4f9. 2024-11-24T22:14:41,927 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0b86957c7b81466a818bbff45fac751f, disabling compactions & flushes 2024-11-24T22:14:41,927 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,kkk,1732486472486.0b86957c7b81466a818bbff45fac751f. 2024-11-24T22:14:41,927 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1732486472486.0b86957c7b81466a818bbff45fac751f. 2024-11-24T22:14:41,928 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1732486472486.0b86957c7b81466a818bbff45fac751f. after waiting 0 ms 2024-11-24T22:14:41,928 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1732486472486.0b86957c7b81466a818bbff45fac751f. 2024-11-24T22:14:41,937 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/0b86957c7b81466a818bbff45fac751f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T22:14:41,937 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,kkk,1732486472486.0b86957c7b81466a818bbff45fac751f. 2024-11-24T22:14:41,938 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0b86957c7b81466a818bbff45fac751f: Waiting for close lock at 1732486481927Running coprocessor pre-close hooks at 1732486481927Disabling compacts and flushes for region at 1732486481927Disabling writes for close at 1732486481928 (+1 ms)Writing region close event to WAL at 1732486481932 (+4 ms)Running coprocessor post-close hooks at 1732486481937 (+5 ms)Closed at 1732486481937 2024-11-24T22:14:41,938 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,kkk,1732486472486.0b86957c7b81466a818bbff45fac751f. 2024-11-24T22:14:41,938 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 83e58b0e0989e11c83283bf7bd41306c, disabling compactions & flushes 2024-11-24T22:14:41,938 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,qqq,1732486472486.83e58b0e0989e11c83283bf7bd41306c. 2024-11-24T22:14:41,938 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1732486472486.83e58b0e0989e11c83283bf7bd41306c. 2024-11-24T22:14:41,938 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1732486472486.83e58b0e0989e11c83283bf7bd41306c. after waiting 0 ms 2024-11-24T22:14:41,938 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1732486472486.83e58b0e0989e11c83283bf7bd41306c. 2024-11-24T22:14:41,941 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/736f44b8f348e2a5a96dfd3abc5e62b2/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T22:14:41,942 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,fff,1732486472486.736f44b8f348e2a5a96dfd3abc5e62b2. 2024-11-24T22:14:41,942 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 736f44b8f348e2a5a96dfd3abc5e62b2: Waiting for close lock at 1732486481926Running coprocessor pre-close hooks at 1732486481926Disabling compacts and flushes for region at 1732486481926Disabling writes for close at 1732486481926Writing region close event to WAL at 1732486481932 (+6 ms)Running coprocessor post-close hooks at 1732486481942 (+10 ms)Closed at 1732486481942 2024-11-24T22:14:41,942 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,fff,1732486472486.736f44b8f348e2a5a96dfd3abc5e62b2. 2024-11-24T22:14:41,942 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing cca6a66154e5c299fad3a247afbfb5fd, disabling compactions & flushes 2024-11-24T22:14:41,942 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ppp,1732486472486.cca6a66154e5c299fad3a247afbfb5fd. 2024-11-24T22:14:41,942 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1732486472486.cca6a66154e5c299fad3a247afbfb5fd. 2024-11-24T22:14:41,942 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1732486472486.cca6a66154e5c299fad3a247afbfb5fd. after waiting 0 ms 2024-11-24T22:14:41,942 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1732486472486.cca6a66154e5c299fad3a247afbfb5fd. 2024-11-24T22:14:41,950 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/83e58b0e0989e11c83283bf7bd41306c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T22:14:41,952 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,qqq,1732486472486.83e58b0e0989e11c83283bf7bd41306c. 2024-11-24T22:14:41,952 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 83e58b0e0989e11c83283bf7bd41306c: Waiting for close lock at 1732486481938Running coprocessor pre-close hooks at 1732486481938Disabling compacts and flushes for region at 1732486481938Disabling writes for close at 1732486481938Writing region close event to WAL at 1732486481945 (+7 ms)Running coprocessor post-close hooks at 1732486481952 (+7 ms)Closed at 1732486481952 2024-11-24T22:14:41,952 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,qqq,1732486472486.83e58b0e0989e11c83283bf7bd41306c. 2024-11-24T22:14:41,952 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c68a44a11eeaccf8a3c2f49ba1446c1e, disabling compactions & flushes 2024-11-24T22:14:41,952 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ccc,1732486472486.c68a44a11eeaccf8a3c2f49ba1446c1e. 2024-11-24T22:14:41,952 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1732486472486.c68a44a11eeaccf8a3c2f49ba1446c1e. 2024-11-24T22:14:41,952 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1732486472486.c68a44a11eeaccf8a3c2f49ba1446c1e. after waiting 0 ms 2024-11-24T22:14:41,952 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1732486472486.c68a44a11eeaccf8a3c2f49ba1446c1e. 2024-11-24T22:14:41,954 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/cca6a66154e5c299fad3a247afbfb5fd/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-24T22:14:41,955 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ppp,1732486472486.cca6a66154e5c299fad3a247afbfb5fd. 2024-11-24T22:14:41,955 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for cca6a66154e5c299fad3a247afbfb5fd: Waiting for close lock at 1732486481942Running coprocessor pre-close hooks at 1732486481942Disabling compacts and flushes for region at 1732486481942Disabling writes for close at 1732486481942Writing region close event to WAL at 1732486481947 (+5 ms)Running coprocessor post-close hooks at 1732486481955 (+8 ms)Closed at 1732486481955 2024-11-24T22:14:41,955 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ppp,1732486472486.cca6a66154e5c299fad3a247afbfb5fd. 2024-11-24T22:14:41,955 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 79c2dd6f654fabefe0fc06b8199e8562, disabling compactions & flushes 2024-11-24T22:14:41,955 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,uuu,1732486472486.79c2dd6f654fabefe0fc06b8199e8562. 2024-11-24T22:14:41,955 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1732486472486.79c2dd6f654fabefe0fc06b8199e8562. 2024-11-24T22:14:41,955 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1732486472486.79c2dd6f654fabefe0fc06b8199e8562. after waiting 0 ms 2024-11-24T22:14:41,955 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1732486472486.79c2dd6f654fabefe0fc06b8199e8562. 2024-11-24T22:14:41,959 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/c68a44a11eeaccf8a3c2f49ba1446c1e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T22:14:41,960 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ccc,1732486472486.c68a44a11eeaccf8a3c2f49ba1446c1e. 2024-11-24T22:14:41,960 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c68a44a11eeaccf8a3c2f49ba1446c1e: Waiting for close lock at 1732486481952Running coprocessor pre-close hooks at 1732486481952Disabling compacts and flushes for region at 1732486481952Disabling writes for close at 1732486481952Writing region close event to WAL at 1732486481953 (+1 ms)Running coprocessor post-close hooks at 1732486481960 (+7 ms)Closed at 1732486481960 2024-11-24T22:14:41,960 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ccc,1732486472486.c68a44a11eeaccf8a3c2f49ba1446c1e. 2024-11-24T22:14:41,960 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ed5b70c013e388180062b1513b980a07, disabling compactions & flushes 2024-11-24T22:14:41,960 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,zzz,1732486472486.ed5b70c013e388180062b1513b980a07. 2024-11-24T22:14:41,960 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1732486472486.ed5b70c013e388180062b1513b980a07. 2024-11-24T22:14:41,960 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1732486472486.ed5b70c013e388180062b1513b980a07. after waiting 0 ms 2024-11-24T22:14:41,960 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1732486472486.ed5b70c013e388180062b1513b980a07. 2024-11-24T22:14:41,977 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/79c2dd6f654fabefe0fc06b8199e8562/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T22:14:41,978 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,uuu,1732486472486.79c2dd6f654fabefe0fc06b8199e8562. 2024-11-24T22:14:41,978 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 79c2dd6f654fabefe0fc06b8199e8562: Waiting for close lock at 1732486481955Running coprocessor pre-close hooks at 1732486481955Disabling compacts and flushes for region at 1732486481955Disabling writes for close at 1732486481955Writing region close event to WAL at 1732486481959 (+4 ms)Running coprocessor post-close hooks at 1732486481978 (+19 ms)Closed at 1732486481978 2024-11-24T22:14:41,978 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,uuu,1732486472486.79c2dd6f654fabefe0fc06b8199e8562. 2024-11-24T22:14:41,978 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing acdab7e426c93b10b34b4d331b21cef0, disabling compactions & flushes 2024-11-24T22:14:41,978 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ttt,1732486472486.acdab7e426c93b10b34b4d331b21cef0. 2024-11-24T22:14:41,978 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1732486472486.acdab7e426c93b10b34b4d331b21cef0. 2024-11-24T22:14:41,978 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1732486472486.acdab7e426c93b10b34b4d331b21cef0. after waiting 0 ms 2024-11-24T22:14:41,978 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1732486472486.acdab7e426c93b10b34b4d331b21cef0. 2024-11-24T22:14:41,986 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/ed5b70c013e388180062b1513b980a07/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T22:14:41,987 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,zzz,1732486472486.ed5b70c013e388180062b1513b980a07. 2024-11-24T22:14:41,988 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ed5b70c013e388180062b1513b980a07: Waiting for close lock at 1732486481960Running coprocessor pre-close hooks at 1732486481960Disabling compacts and flushes for region at 1732486481960Disabling writes for close at 1732486481960Writing region close event to WAL at 1732486481963 (+3 ms)Running coprocessor post-close hooks at 1732486481987 (+24 ms)Closed at 1732486481987 2024-11-24T22:14:41,988 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,zzz,1732486472486.ed5b70c013e388180062b1513b980a07. 2024-11-24T22:14:41,988 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 50d9b38af2d6413e500860133d150aa8, disabling compactions & flushes 2024-11-24T22:14:41,988 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,www,1732486472486.50d9b38af2d6413e500860133d150aa8. 2024-11-24T22:14:41,988 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1732486472486.50d9b38af2d6413e500860133d150aa8. 2024-11-24T22:14:41,988 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,www,1732486472486.50d9b38af2d6413e500860133d150aa8. after waiting 0 ms 2024-11-24T22:14:41,988 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,www,1732486472486.50d9b38af2d6413e500860133d150aa8. 2024-11-24T22:14:42,001 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/acdab7e426c93b10b34b4d331b21cef0/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T22:14:42,002 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ttt,1732486472486.acdab7e426c93b10b34b4d331b21cef0. 2024-11-24T22:14:42,002 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for acdab7e426c93b10b34b4d331b21cef0: Waiting for close lock at 1732486481978Running coprocessor pre-close hooks at 1732486481978Disabling compacts and flushes for region at 1732486481978Disabling writes for close at 1732486481978Writing region close event to WAL at 1732486481992 (+14 ms)Running coprocessor post-close hooks at 1732486482002 (+10 ms)Closed at 1732486482002 2024-11-24T22:14:42,002 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ttt,1732486472486.acdab7e426c93b10b34b4d331b21cef0. 2024-11-24T22:14:42,003 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/50d9b38af2d6413e500860133d150aa8/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T22:14:42,004 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,www,1732486472486.50d9b38af2d6413e500860133d150aa8. 2024-11-24T22:14:42,004 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 50d9b38af2d6413e500860133d150aa8: Waiting for close lock at 1732486481988Running coprocessor pre-close hooks at 1732486481988Disabling compacts and flushes for region at 1732486481988Disabling writes for close at 1732486481988Writing region close event to WAL at 1732486481997 (+9 ms)Running coprocessor post-close hooks at 1732486482004 (+7 ms)Closed at 1732486482004 2024-11-24T22:14:42,004 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,www,1732486472486.50d9b38af2d6413e500860133d150aa8. 2024-11-24T22:14:42,005 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing dc147c653b22684b8e41092cd6a0c8eb, disabling compactions & flushes 2024-11-24T22:14:42,005 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,mmm,1732486472486.dc147c653b22684b8e41092cd6a0c8eb. 2024-11-24T22:14:42,005 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1732486472486.dc147c653b22684b8e41092cd6a0c8eb. 2024-11-24T22:14:42,005 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1732486472486.dc147c653b22684b8e41092cd6a0c8eb. after waiting 0 ms 2024-11-24T22:14:42,005 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1732486472486.dc147c653b22684b8e41092cd6a0c8eb. 2024-11-24T22:14:42,006 INFO [RS:1;bd6592c37bb2:38125 {}] regionserver.HRegionServer(976): stopping server bd6592c37bb2,38125,1732486469405; all regions closed. 2024-11-24T22:14:42,008 DEBUG [RS:2;bd6592c37bb2:42639 {}] regionserver.HRegionServer(1351): Waiting on dc147c653b22684b8e41092cd6a0c8eb, fc9f96fd4c44240a85e6b2c001e46153 2024-11-24T22:14:42,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34013 is added to blk_1073741835_1011 (size=6805) 2024-11-24T22:14:42,022 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/dc147c653b22684b8e41092cd6a0c8eb/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T22:14:42,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35383 is added to blk_1073741835_1011 (size=6805) 2024-11-24T22:14:42,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741835_1011 (size=6805) 2024-11-24T22:14:42,024 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,mmm,1732486472486.dc147c653b22684b8e41092cd6a0c8eb. 2024-11-24T22:14:42,024 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for dc147c653b22684b8e41092cd6a0c8eb: Waiting for close lock at 1732486482005Running coprocessor pre-close hooks at 1732486482005Disabling compacts and flushes for region at 1732486482005Disabling writes for close at 1732486482005Writing region close event to WAL at 1732486482007 (+2 ms)Running coprocessor post-close hooks at 1732486482024 (+17 ms)Closed at 1732486482024 2024-11-24T22:14:42,024 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,mmm,1732486472486.dc147c653b22684b8e41092cd6a0c8eb. 2024-11-24T22:14:42,024 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fc9f96fd4c44240a85e6b2c001e46153, disabling compactions & flushes 2024-11-24T22:14:42,024 DEBUG [RS:1;bd6592c37bb2:38125 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:42,024 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,vvv,1732486472486.fc9f96fd4c44240a85e6b2c001e46153. 2024-11-24T22:14:42,024 INFO [RS:1;bd6592c37bb2:38125 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T22:14:42,024 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1732486472486.fc9f96fd4c44240a85e6b2c001e46153. 2024-11-24T22:14:42,024 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1732486472486.fc9f96fd4c44240a85e6b2c001e46153. after waiting 0 ms 2024-11-24T22:14:42,024 INFO [RS:1;bd6592c37bb2:38125 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T22:14:42,024 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1732486472486.fc9f96fd4c44240a85e6b2c001e46153. 2024-11-24T22:14:42,025 INFO [RS:1;bd6592c37bb2:38125 {}] hbase.ChoreService(370): Chore service for: regionserver/bd6592c37bb2:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-24T22:14:42,025 INFO [RS:1;bd6592c37bb2:38125 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T22:14:42,025 INFO [RS:1;bd6592c37bb2:38125 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T22:14:42,025 INFO [RS:1;bd6592c37bb2:38125 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T22:14:42,025 INFO [RS:1;bd6592c37bb2:38125 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T22:14:42,025 INFO [RS:1;bd6592c37bb2:38125 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38125 2024-11-24T22:14:42,026 INFO [regionserver/bd6592c37bb2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T22:14:42,028 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36279-0x1014c0b79ef0000, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T22:14:42,028 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38125-0x1014c0b79ef0002, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/bd6592c37bb2,38125,1732486469405 2024-11-24T22:14:42,028 INFO [RS:1;bd6592c37bb2:38125 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T22:14:42,029 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [bd6592c37bb2,38125,1732486469405] 2024-11-24T22:14:42,032 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/bd6592c37bb2,38125,1732486469405 already deleted, retry=false 2024-11-24T22:14:42,032 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; bd6592c37bb2,38125,1732486469405 expired; onlineServers=1 2024-11-24T22:14:42,032 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/data/default/test/fc9f96fd4c44240a85e6b2c001e46153/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T22:14:42,034 INFO [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,vvv,1732486472486.fc9f96fd4c44240a85e6b2c001e46153. 2024-11-24T22:14:42,034 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fc9f96fd4c44240a85e6b2c001e46153: Waiting for close lock at 1732486482024Running coprocessor pre-close hooks at 1732486482024Disabling compacts and flushes for region at 1732486482024Disabling writes for close at 1732486482024Writing region close event to WAL at 1732486482025 (+1 ms)Running coprocessor post-close hooks at 1732486482034 (+9 ms)Closed at 1732486482034 2024-11-24T22:14:42,034 DEBUG [RS_CLOSE_REGION-regionserver/bd6592c37bb2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,vvv,1732486472486.fc9f96fd4c44240a85e6b2c001e46153. 2024-11-24T22:14:42,131 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38125-0x1014c0b79ef0002, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T22:14:42,131 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38125-0x1014c0b79ef0002, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T22:14:42,131 INFO [RS:1;bd6592c37bb2:38125 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T22:14:42,132 INFO [RS:1;bd6592c37bb2:38125 {}] regionserver.HRegionServer(1031): Exiting; stopping=bd6592c37bb2,38125,1732486469405; zookeeper connection closed. 2024-11-24T22:14:42,132 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7d59ece6 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7d59ece6 2024-11-24T22:14:42,208 INFO [RS:2;bd6592c37bb2:42639 {}] regionserver.HRegionServer(976): stopping server bd6592c37bb2,42639,1732486469476; all regions closed. 2024-11-24T22:14:42,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34013 is added to blk_1073741865_1041 (size=4192) 2024-11-24T22:14:42,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741865_1041 (size=4192) 2024-11-24T22:14:42,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35383 is added to blk_1073741865_1041 (size=4192) 2024-11-24T22:14:42,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34013 is added to blk_1073741833_1009 (size=6800) 2024-11-24T22:14:42,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35383 is added to blk_1073741833_1009 (size=6800) 2024-11-24T22:14:42,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741833_1009 (size=6800) 2024-11-24T22:14:42,247 DEBUG [RS:2;bd6592c37bb2:42639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T22:14:42,247 INFO [RS:2;bd6592c37bb2:42639 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T22:14:42,247 INFO [RS:2;bd6592c37bb2:42639 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T22:14:42,247 INFO [RS:2;bd6592c37bb2:42639 {}] hbase.ChoreService(370): Chore service for: regionserver/bd6592c37bb2:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-24T22:14:42,248 INFO [RS:2;bd6592c37bb2:42639 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T22:14:42,248 INFO [RS:2;bd6592c37bb2:42639 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42639 2024-11-24T22:14:42,249 INFO [regionserver/bd6592c37bb2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T22:14:42,250 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42639-0x1014c0b79ef0003, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/bd6592c37bb2,42639,1732486469476 2024-11-24T22:14:42,250 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36279-0x1014c0b79ef0000, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T22:14:42,250 INFO [RS:2;bd6592c37bb2:42639 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T22:14:42,251 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [bd6592c37bb2,42639,1732486469476] 2024-11-24T22:14:42,253 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/bd6592c37bb2,42639,1732486469476 already deleted, retry=false 2024-11-24T22:14:42,253 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; bd6592c37bb2,42639,1732486469476 expired; onlineServers=0 2024-11-24T22:14:42,253 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'bd6592c37bb2,36279,1732486468255' ***** 2024-11-24T22:14:42,253 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-24T22:14:42,253 INFO [M:0;bd6592c37bb2:36279 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T22:14:42,253 INFO [M:0;bd6592c37bb2:36279 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T22:14:42,253 DEBUG [M:0;bd6592c37bb2:36279 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-24T22:14:42,254 DEBUG [M:0;bd6592c37bb2:36279 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-24T22:14:42,254 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster-HFileCleaner.small.0-1732486470873 {}] cleaner.HFileCleaner(306): Exit Thread[master/bd6592c37bb2:0:becomeActiveMaster-HFileCleaner.small.0-1732486470873,5,FailOnTimeoutGroup] 2024-11-24T22:14:42,254 INFO [M:0;bd6592c37bb2:36279 {}] hbase.ChoreService(370): Chore service for: master/bd6592c37bb2:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-24T22:14:42,254 INFO [M:0;bd6592c37bb2:36279 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T22:14:42,254 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-24T22:14:42,254 DEBUG [M:0;bd6592c37bb2:36279 {}] master.HMaster(1795): Stopping service threads 2024-11-24T22:14:42,254 INFO [M:0;bd6592c37bb2:36279 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-24T22:14:42,254 INFO [M:0;bd6592c37bb2:36279 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T22:14:42,254 DEBUG [master/bd6592c37bb2:0:becomeActiveMaster-HFileCleaner.large.0-1732486470860 {}] cleaner.HFileCleaner(306): Exit Thread[master/bd6592c37bb2:0:becomeActiveMaster-HFileCleaner.large.0-1732486470860,5,FailOnTimeoutGroup] 2024-11-24T22:14:42,255 INFO [M:0;bd6592c37bb2:36279 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T22:14:42,255 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-24T22:14:42,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36279-0x1014c0b79ef0000, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-24T22:14:42,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36279-0x1014c0b79ef0000, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T22:14:42,258 DEBUG [M:0;bd6592c37bb2:36279 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/master already deleted, retry=false 2024-11-24T22:14:42,258 DEBUG [M:0;bd6592c37bb2:36279 {}] master.ActiveMasterManager(353): master:36279-0x1014c0b79ef0000, quorum=127.0.0.1:56583, baseZNode=/hbase Failed delete of our master address node; KeeperErrorCode = NoNode for /hbase/master 2024-11-24T22:14:42,259 INFO [M:0;bd6592c37bb2:36279 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/.lastflushedseqids 2024-11-24T22:14:42,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34013 is added to blk_1073741870_1046 (size=1696) 2024-11-24T22:14:42,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35383 is added to blk_1073741870_1046 (size=1696) 2024-11-24T22:14:42,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741870_1046 (size=1696) 2024-11-24T22:14:42,295 INFO [M:0;bd6592c37bb2:36279 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T22:14:42,295 INFO [M:0;bd6592c37bb2:36279 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-24T22:14:42,295 DEBUG [M:0;bd6592c37bb2:36279 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T22:14:42,295 INFO [M:0;bd6592c37bb2:36279 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T22:14:42,295 DEBUG [M:0;bd6592c37bb2:36279 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T22:14:42,295 DEBUG [M:0;bd6592c37bb2:36279 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T22:14:42,295 DEBUG [M:0;bd6592c37bb2:36279 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T22:14:42,295 INFO [M:0;bd6592c37bb2:36279 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=258.81 KB heapSize=314.57 KB 2024-11-24T22:14:42,319 DEBUG [M:0;bd6592c37bb2:36279 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/eaf0dd766b8740099ea0a928139dee5c is 82, key is hbase:meta,,1/info:regioninfo/1732486477920/Put/seqid=0 2024-11-24T22:14:42,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35383 is added to blk_1073741871_1047 (size=6071) 2024-11-24T22:14:42,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34013 is added to blk_1073741871_1047 (size=6071) 2024-11-24T22:14:42,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34899 is added to blk_1073741871_1047 (size=6071) 2024-11-24T22:14:42,328 INFO [M:0;bd6592c37bb2:36279 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=597 (bloomFilter=true), to=hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/eaf0dd766b8740099ea0a928139dee5c 2024-11-24T22:14:42,352 INFO [RS:2;bd6592c37bb2:42639 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T22:14:42,352 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42639-0x1014c0b79ef0003, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T22:14:42,352 INFO [RS:2;bd6592c37bb2:42639 {}] regionserver.HRegionServer(1031): Exiting; stopping=bd6592c37bb2,42639,1732486469476; zookeeper connection closed. 2024-11-24T22:14:42,352 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42639-0x1014c0b79ef0003, quorum=127.0.0.1:56583, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T22:14:42,353 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6890001a {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6890001a 2024-11-24T22:14:42,353 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-24T22:14:42,358 DEBUG [M:0;bd6592c37bb2:36279 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36281/user/jenkins/test-data/dee1efa5-6d54-4abd-f878-b9cd51170f7c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c44afc0cb2264b38b01fdd751cf4ed0e is 2987, key is \x00\x00\x00\x00\x00\x00\x00