2024-11-15 22:14:19,088 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-15 22:14:19,106 main DEBUG Took 0.015248 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-15 22:14:19,106 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-15 22:14:19,106 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-15 22:14:19,107 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-15 22:14:19,109 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 22:14:19,120 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-15 22:14:19,140 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 22:14:19,142 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 22:14:19,143 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 22:14:19,143 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 22:14:19,144 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 22:14:19,145 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 22:14:19,146 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 22:14:19,146 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 22:14:19,147 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 22:14:19,148 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 22:14:19,149 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 22:14:19,149 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 22:14:19,150 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 22:14:19,151 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 22:14:19,152 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 22:14:19,152 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 22:14:19,153 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 22:14:19,153 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 22:14:19,154 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 22:14:19,154 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 22:14:19,155 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 22:14:19,156 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 22:14:19,158 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 22:14:19,159 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 22:14:19,159 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 22:14:19,159 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-15 22:14:19,161 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 22:14:19,163 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-15 22:14:19,166 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-15 22:14:19,167 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-15 22:14:19,168 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-15 22:14:19,168 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-15 22:14:19,177 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-15 22:14:19,180 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-15 22:14:19,182 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-15 22:14:19,182 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-15 22:14:19,182 main DEBUG createAppenders(={Console}) 2024-11-15 22:14:19,183 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 initialized 2024-11-15 22:14:19,183 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-15 22:14:19,184 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 OK. 2024-11-15 22:14:19,184 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-15 22:14:19,184 main DEBUG OutputStream closed 2024-11-15 22:14:19,185 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-15 22:14:19,185 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-15 22:14:19,185 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@c88a337 OK 2024-11-15 22:14:19,285 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-15 22:14:19,290 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-15 22:14:19,292 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-15 22:14:19,294 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-15 22:14:19,294 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-15 22:14:19,295 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-15 22:14:19,295 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-15 22:14:19,296 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-15 22:14:19,296 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-15 22:14:19,297 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-15 22:14:19,297 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-15 22:14:19,298 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-15 22:14:19,298 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-15 22:14:19,299 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-15 22:14:19,299 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-15 22:14:19,299 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-15 22:14:19,300 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-15 22:14:19,301 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-15 22:14:19,304 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-15 22:14:19,305 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@13330ac6) with optional ClassLoader: null 2024-11-15 22:14:19,305 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-15 22:14:19,306 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@13330ac6] started OK. 2024-11-15T22:14:19,323 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP timeout: 13 mins 2024-11-15 22:14:19,327 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-15 22:14:19,327 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-15T22:14:19,640 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f42a1aa2-5628-94c3-94cd-cb342e5fdc53 2024-11-15T22:14:19,668 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$HMasterForTest, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-15T22:14:19,696 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f42a1aa2-5628-94c3-94cd-cb342e5fdc53/cluster_89ff9f97-76ad-0edf-ec0b-644decd15cd1, deleteOnExit=true 2024-11-15T22:14:19,697 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-15T22:14:19,699 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f42a1aa2-5628-94c3-94cd-cb342e5fdc53/test.cache.data in system properties and HBase conf 2024-11-15T22:14:19,700 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f42a1aa2-5628-94c3-94cd-cb342e5fdc53/hadoop.tmp.dir in system properties and HBase conf 2024-11-15T22:14:19,701 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f42a1aa2-5628-94c3-94cd-cb342e5fdc53/hadoop.log.dir in system properties and HBase conf 2024-11-15T22:14:19,704 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f42a1aa2-5628-94c3-94cd-cb342e5fdc53/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-15T22:14:19,705 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f42a1aa2-5628-94c3-94cd-cb342e5fdc53/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-15T22:14:19,705 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-15T22:14:19,842 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-15T22:14:19,979 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-15T22:14:19,984 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f42a1aa2-5628-94c3-94cd-cb342e5fdc53/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-15T22:14:19,985 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f42a1aa2-5628-94c3-94cd-cb342e5fdc53/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-15T22:14:19,986 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f42a1aa2-5628-94c3-94cd-cb342e5fdc53/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-15T22:14:19,986 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f42a1aa2-5628-94c3-94cd-cb342e5fdc53/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-15T22:14:19,987 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f42a1aa2-5628-94c3-94cd-cb342e5fdc53/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-15T22:14:19,987 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f42a1aa2-5628-94c3-94cd-cb342e5fdc53/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-15T22:14:19,988 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f42a1aa2-5628-94c3-94cd-cb342e5fdc53/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-15T22:14:19,989 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f42a1aa2-5628-94c3-94cd-cb342e5fdc53/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-15T22:14:19,989 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f42a1aa2-5628-94c3-94cd-cb342e5fdc53/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-15T22:14:19,990 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f42a1aa2-5628-94c3-94cd-cb342e5fdc53/nfs.dump.dir in system properties and HBase conf 2024-11-15T22:14:19,991 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f42a1aa2-5628-94c3-94cd-cb342e5fdc53/java.io.tmpdir in system properties and HBase conf 2024-11-15T22:14:19,991 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f42a1aa2-5628-94c3-94cd-cb342e5fdc53/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-15T22:14:19,992 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f42a1aa2-5628-94c3-94cd-cb342e5fdc53/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-15T22:14:19,993 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f42a1aa2-5628-94c3-94cd-cb342e5fdc53/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-15T22:14:21,802 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-15T22:14:21,920 INFO [Time-limited test {}] log.Log(170): Logging initialized @3771ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-15T22:14:22,025 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-15T22:14:22,138 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-15T22:14:22,180 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-15T22:14:22,181 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-15T22:14:22,183 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-15T22:14:22,202 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-15T22:14:22,210 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@952689a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f42a1aa2-5628-94c3-94cd-cb342e5fdc53/hadoop.log.dir/,AVAILABLE} 2024-11-15T22:14:22,211 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@378c34a2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-15T22:14:22,498 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@302cfe51{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f42a1aa2-5628-94c3-94cd-cb342e5fdc53/java.io.tmpdir/jetty-localhost-41513-hadoop-hdfs-3_4_1-tests_jar-_-any-16388713133834585518/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-15T22:14:22,520 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@c1eb5a9{HTTP/1.1, (http/1.1)}{localhost:41513} 2024-11-15T22:14:22,521 INFO [Time-limited test {}] server.Server(415): Started @4373ms 2024-11-15T22:14:23,192 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-15T22:14:23,203 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-15T22:14:23,212 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-15T22:14:23,213 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-15T22:14:23,213 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-15T22:14:23,233 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1bbc6100{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f42a1aa2-5628-94c3-94cd-cb342e5fdc53/hadoop.log.dir/,AVAILABLE} 2024-11-15T22:14:23,234 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7de7e807{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-15T22:14:23,399 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6ff280d6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f42a1aa2-5628-94c3-94cd-cb342e5fdc53/java.io.tmpdir/jetty-localhost-37647-hadoop-hdfs-3_4_1-tests_jar-_-any-14331446202902389499/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-15T22:14:23,400 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6bbcda89{HTTP/1.1, (http/1.1)}{localhost:37647} 2024-11-15T22:14:23,400 INFO [Time-limited test {}] server.Server(415): Started @5253ms 2024-11-15T22:14:23,473 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-15T22:14:23,782 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-15T22:14:23,807 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-15T22:14:23,861 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-15T22:14:23,861 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-15T22:14:23,861 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-15T22:14:23,863 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@49a00864{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f42a1aa2-5628-94c3-94cd-cb342e5fdc53/hadoop.log.dir/,AVAILABLE} 2024-11-15T22:14:23,863 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7e78dc0c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-15T22:14:24,048 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@16ce28b8{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f42a1aa2-5628-94c3-94cd-cb342e5fdc53/java.io.tmpdir/jetty-localhost-38983-hadoop-hdfs-3_4_1-tests_jar-_-any-600760088547799104/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-15T22:14:24,053 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@313e9338{HTTP/1.1, (http/1.1)}{localhost:38983} 2024-11-15T22:14:24,054 INFO [Time-limited test {}] server.Server(415): Started @5906ms 2024-11-15T22:14:24,065 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-15T22:14:24,233 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-15T22:14:24,246 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-15T22:14:24,253 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-15T22:14:24,253 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-15T22:14:24,253 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-15T22:14:24,257 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@33fcc865{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f42a1aa2-5628-94c3-94cd-cb342e5fdc53/hadoop.log.dir/,AVAILABLE} 2024-11-15T22:14:24,258 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@76da50c3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-15T22:14:24,435 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@744135cb{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f42a1aa2-5628-94c3-94cd-cb342e5fdc53/java.io.tmpdir/jetty-localhost-37039-hadoop-hdfs-3_4_1-tests_jar-_-any-15674343567776689676/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-15T22:14:24,437 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5b3e09e4{HTTP/1.1, (http/1.1)}{localhost:37039} 2024-11-15T22:14:24,437 INFO [Time-limited test {}] server.Server(415): Started @6290ms 2024-11-15T22:14:24,441 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-15T22:14:25,779 WARN [Thread-121 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f42a1aa2-5628-94c3-94cd-cb342e5fdc53/cluster_89ff9f97-76ad-0edf-ec0b-644decd15cd1/data/data3/current/BP-1548789118-172.17.0.3-1731708861150/current, will proceed with Du for space computation calculation, 2024-11-15T22:14:25,779 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f42a1aa2-5628-94c3-94cd-cb342e5fdc53/cluster_89ff9f97-76ad-0edf-ec0b-644decd15cd1/data/data4/current/BP-1548789118-172.17.0.3-1731708861150/current, will proceed with Du for space computation calculation, 2024-11-15T22:14:25,781 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f42a1aa2-5628-94c3-94cd-cb342e5fdc53/cluster_89ff9f97-76ad-0edf-ec0b-644decd15cd1/data/data1/current/BP-1548789118-172.17.0.3-1731708861150/current, will proceed with Du for space computation calculation, 2024-11-15T22:14:25,785 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f42a1aa2-5628-94c3-94cd-cb342e5fdc53/cluster_89ff9f97-76ad-0edf-ec0b-644decd15cd1/data/data2/current/BP-1548789118-172.17.0.3-1731708861150/current, will proceed with Du for space computation calculation, 2024-11-15T22:14:25,823 WARN [Thread-132 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f42a1aa2-5628-94c3-94cd-cb342e5fdc53/cluster_89ff9f97-76ad-0edf-ec0b-644decd15cd1/data/data5/current/BP-1548789118-172.17.0.3-1731708861150/current, will proceed with Du for space computation calculation, 2024-11-15T22:14:25,823 WARN [Thread-133 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f42a1aa2-5628-94c3-94cd-cb342e5fdc53/cluster_89ff9f97-76ad-0edf-ec0b-644decd15cd1/data/data6/current/BP-1548789118-172.17.0.3-1731708861150/current, will proceed with Du for space computation calculation, 2024-11-15T22:14:25,863 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-15T22:14:25,863 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-15T22:14:25,868 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-15T22:14:25,960 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd0165f4193cc38e with lease ID 0xa61786e6a7dd08a4: Processing first storage report for DS-46524d7c-f751-41cf-9f4a-9f083a5e2adc from datanode DatanodeRegistration(127.0.0.1:33631, datanodeUuid=6a2b3b8e-9ba4-4191-91a9-4bbc45ed5f85, infoPort=45219, infoSecurePort=0, ipcPort=40585, storageInfo=lv=-57;cid=testClusterID;nsid=655564362;c=1731708861150) 2024-11-15T22:14:25,962 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd0165f4193cc38e with lease ID 0xa61786e6a7dd08a4: from storage DS-46524d7c-f751-41cf-9f4a-9f083a5e2adc node DatanodeRegistration(127.0.0.1:33631, datanodeUuid=6a2b3b8e-9ba4-4191-91a9-4bbc45ed5f85, infoPort=45219, infoSecurePort=0, ipcPort=40585, storageInfo=lv=-57;cid=testClusterID;nsid=655564362;c=1731708861150), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-15T22:14:25,962 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x126232039e464e4c with lease ID 0xa61786e6a7dd08a2: Processing first storage report for DS-b38187b6-2a6b-449a-a41a-28ace83252fc from datanode DatanodeRegistration(127.0.0.1:39269, datanodeUuid=fe542ff9-f650-4c22-979d-6fce7732abec, infoPort=41859, infoSecurePort=0, ipcPort=32941, storageInfo=lv=-57;cid=testClusterID;nsid=655564362;c=1731708861150) 2024-11-15T22:14:25,962 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x126232039e464e4c with lease ID 0xa61786e6a7dd08a2: from storage DS-b38187b6-2a6b-449a-a41a-28ace83252fc node DatanodeRegistration(127.0.0.1:39269, datanodeUuid=fe542ff9-f650-4c22-979d-6fce7732abec, infoPort=41859, infoSecurePort=0, ipcPort=32941, storageInfo=lv=-57;cid=testClusterID;nsid=655564362;c=1731708861150), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-15T22:14:25,963 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc091498f6a090ecb with lease ID 0xa61786e6a7dd08a3: Processing first storage report for DS-0e1f6fcf-e559-4b2f-8ba2-2f7f76775d19 from datanode DatanodeRegistration(127.0.0.1:40641, datanodeUuid=34aebda6-5d38-4096-b561-27465b75199e, infoPort=34309, infoSecurePort=0, ipcPort=36257, storageInfo=lv=-57;cid=testClusterID;nsid=655564362;c=1731708861150) 2024-11-15T22:14:25,963 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc091498f6a090ecb with lease ID 0xa61786e6a7dd08a3: from storage DS-0e1f6fcf-e559-4b2f-8ba2-2f7f76775d19 node DatanodeRegistration(127.0.0.1:40641, datanodeUuid=34aebda6-5d38-4096-b561-27465b75199e, infoPort=34309, infoSecurePort=0, ipcPort=36257, storageInfo=lv=-57;cid=testClusterID;nsid=655564362;c=1731708861150), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-15T22:14:25,963 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd0165f4193cc38e with lease ID 0xa61786e6a7dd08a4: Processing first storage report for DS-62462141-391b-4bca-be0a-2819e935296b from datanode DatanodeRegistration(127.0.0.1:33631, datanodeUuid=6a2b3b8e-9ba4-4191-91a9-4bbc45ed5f85, infoPort=45219, infoSecurePort=0, ipcPort=40585, storageInfo=lv=-57;cid=testClusterID;nsid=655564362;c=1731708861150) 2024-11-15T22:14:25,963 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd0165f4193cc38e with lease ID 0xa61786e6a7dd08a4: from storage DS-62462141-391b-4bca-be0a-2819e935296b node DatanodeRegistration(127.0.0.1:33631, datanodeUuid=6a2b3b8e-9ba4-4191-91a9-4bbc45ed5f85, infoPort=45219, infoSecurePort=0, ipcPort=40585, storageInfo=lv=-57;cid=testClusterID;nsid=655564362;c=1731708861150), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-15T22:14:25,964 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc091498f6a090ecb with lease ID 0xa61786e6a7dd08a3: Processing first storage report for DS-9acb5303-47a0-4d0d-aa50-ea537ec46bd4 from datanode DatanodeRegistration(127.0.0.1:40641, datanodeUuid=34aebda6-5d38-4096-b561-27465b75199e, infoPort=34309, infoSecurePort=0, ipcPort=36257, storageInfo=lv=-57;cid=testClusterID;nsid=655564362;c=1731708861150) 2024-11-15T22:14:25,964 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc091498f6a090ecb with lease ID 0xa61786e6a7dd08a3: from storage DS-9acb5303-47a0-4d0d-aa50-ea537ec46bd4 node DatanodeRegistration(127.0.0.1:40641, datanodeUuid=34aebda6-5d38-4096-b561-27465b75199e, infoPort=34309, infoSecurePort=0, ipcPort=36257, storageInfo=lv=-57;cid=testClusterID;nsid=655564362;c=1731708861150), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-15T22:14:25,966 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x126232039e464e4c with lease ID 0xa61786e6a7dd08a2: Processing first storage report for DS-059fd3e0-f5e8-4ef3-a7d3-226d983028a8 from datanode DatanodeRegistration(127.0.0.1:39269, datanodeUuid=fe542ff9-f650-4c22-979d-6fce7732abec, infoPort=41859, infoSecurePort=0, ipcPort=32941, storageInfo=lv=-57;cid=testClusterID;nsid=655564362;c=1731708861150) 2024-11-15T22:14:25,967 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x126232039e464e4c with lease ID 0xa61786e6a7dd08a2: from storage DS-059fd3e0-f5e8-4ef3-a7d3-226d983028a8 node DatanodeRegistration(127.0.0.1:39269, datanodeUuid=fe542ff9-f650-4c22-979d-6fce7732abec, infoPort=41859, infoSecurePort=0, ipcPort=32941, storageInfo=lv=-57;cid=testClusterID;nsid=655564362;c=1731708861150), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-15T22:14:25,997 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f42a1aa2-5628-94c3-94cd-cb342e5fdc53 2024-11-15T22:14:26,125 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f42a1aa2-5628-94c3-94cd-cb342e5fdc53/cluster_89ff9f97-76ad-0edf-ec0b-644decd15cd1/zookeeper_0, clientPort=60621, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f42a1aa2-5628-94c3-94cd-cb342e5fdc53/cluster_89ff9f97-76ad-0edf-ec0b-644decd15cd1/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f42a1aa2-5628-94c3-94cd-cb342e5fdc53/cluster_89ff9f97-76ad-0edf-ec0b-644decd15cd1/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-15T22:14:26,139 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60621 2024-11-15T22:14:26,158 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T22:14:26,162 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T22:14:26,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741825_1001 (size=7) 2024-11-15T22:14:26,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39269 is added to blk_1073741825_1001 (size=7) 2024-11-15T22:14:26,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33631 is added to blk_1073741825_1001 (size=7) 2024-11-15T22:14:26,914 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa with version=8 2024-11-15T22:14:26,915 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/hbase-staging 2024-11-15T22:14:27,003 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-15T22:14:27,345 INFO [Time-limited test {}] client.ConnectionUtils(128): master/e611192d6313:0 server-side Connection retries=45 2024-11-15T22:14:27,358 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T22:14:27,359 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-15T22:14:27,366 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-15T22:14:27,366 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T22:14:27,366 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-15T22:14:27,545 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-15T22:14:27,621 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-15T22:14:27,633 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-15T22:14:27,636 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T22:14:27,668 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 3748 (auto-detected) 2024-11-15T22:14:27,670 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-15T22:14:27,689 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:39087 2024-11-15T22:14:27,715 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39087 connecting to ZooKeeper ensemble=127.0.0.1:60621 2024-11-15T22:14:27,841 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:390870x0, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-15T22:14:27,846 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39087-0x101409049d30000 connected 2024-11-15T22:14:27,944 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T22:14:27,948 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T22:14:27,964 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39087-0x101409049d30000, quorum=127.0.0.1:60621, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T22:14:27,969 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa, hbase.cluster.distributed=false 2024-11-15T22:14:27,999 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39087-0x101409049d30000, quorum=127.0.0.1:60621, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-15T22:14:28,012 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39087 2024-11-15T22:14:28,015 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39087 2024-11-15T22:14:28,016 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39087 2024-11-15T22:14:28,020 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39087 2024-11-15T22:14:28,021 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39087 2024-11-15T22:14:28,350 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/e611192d6313:0 server-side Connection retries=45 2024-11-15T22:14:28,353 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T22:14:28,353 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-15T22:14:28,354 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-15T22:14:28,354 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T22:14:28,354 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-15T22:14:28,359 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-15T22:14:28,363 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T22:14:28,364 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:38729 2024-11-15T22:14:28,367 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38729 connecting to ZooKeeper ensemble=127.0.0.1:60621 2024-11-15T22:14:28,369 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T22:14:28,376 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T22:14:28,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:387290x0, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-15T22:14:28,501 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:387290x0, quorum=127.0.0.1:60621, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T22:14:28,502 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38729-0x101409049d30001 connected 2024-11-15T22:14:28,682 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-15T22:14:28,920 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-15T22:14:28,922 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38729-0x101409049d30001, quorum=127.0.0.1:60621, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-15T22:14:29,007 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38729-0x101409049d30001, quorum=127.0.0.1:60621, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-15T22:14:29,008 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38729 2024-11-15T22:14:29,012 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38729 2024-11-15T22:14:29,014 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38729 2024-11-15T22:14:29,016 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38729 2024-11-15T22:14:29,017 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38729 2024-11-15T22:14:29,042 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/e611192d6313:0 server-side Connection retries=45 2024-11-15T22:14:29,042 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T22:14:29,043 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-15T22:14:29,043 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-15T22:14:29,043 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T22:14:29,044 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-15T22:14:29,044 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-15T22:14:29,044 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T22:14:29,049 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:41231 2024-11-15T22:14:29,051 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41231 connecting to ZooKeeper ensemble=127.0.0.1:60621 2024-11-15T22:14:29,053 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T22:14:29,058 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T22:14:29,333 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:412310x0, quorum=127.0.0.1:60621, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T22:14:29,334 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-15T22:14:29,335 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-15T22:14:29,337 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:412310x0, quorum=127.0.0.1:60621, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-15T22:14:29,339 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:412310x0, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-15T22:14:29,341 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:412310x0, quorum=127.0.0.1:60621, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-15T22:14:29,342 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41231-0x101409049d30002 connected 2024-11-15T22:14:29,342 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41231 2024-11-15T22:14:29,343 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41231 2024-11-15T22:14:29,343 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41231 2024-11-15T22:14:29,344 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41231 2024-11-15T22:14:29,344 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41231 2024-11-15T22:14:29,386 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/e611192d6313:0 server-side Connection retries=45 2024-11-15T22:14:29,386 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T22:14:29,386 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-15T22:14:29,387 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-15T22:14:29,387 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T22:14:29,387 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-15T22:14:29,387 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-15T22:14:29,388 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T22:14:29,395 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:43349 2024-11-15T22:14:29,398 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43349 connecting to ZooKeeper ensemble=127.0.0.1:60621 2024-11-15T22:14:29,401 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T22:14:29,407 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T22:14:29,541 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:433490x0, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-15T22:14:29,542 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:433490x0, quorum=127.0.0.1:60621, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T22:14:29,543 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-15T22:14:29,545 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43349-0x101409049d30003 connected 2024-11-15T22:14:29,546 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-15T22:14:29,547 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43349-0x101409049d30003, quorum=127.0.0.1:60621, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-15T22:14:29,550 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43349-0x101409049d30003, quorum=127.0.0.1:60621, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-15T22:14:29,557 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43349 2024-11-15T22:14:29,558 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43349 2024-11-15T22:14:29,560 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43349 2024-11-15T22:14:29,565 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43349 2024-11-15T22:14:29,565 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43349 2024-11-15T22:14:29,585 DEBUG [M:0;e611192d6313:39087 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;e611192d6313:39087 2024-11-15T22:14:29,586 INFO [master/e611192d6313:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/e611192d6313,39087,1731708867142 2024-11-15T22:14:29,908 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x101409049d30001, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T22:14:29,908 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41231-0x101409049d30002, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T22:14:29,908 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39087-0x101409049d30000, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T22:14:29,908 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43349-0x101409049d30003, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T22:14:29,910 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39087-0x101409049d30000, quorum=127.0.0.1:60621, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/e611192d6313,39087,1731708867142 2024-11-15T22:14:30,221 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x101409049d30001, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-15T22:14:30,221 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41231-0x101409049d30002, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-15T22:14:30,221 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39087-0x101409049d30000, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T22:14:30,221 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43349-0x101409049d30003, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-15T22:14:30,221 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43349-0x101409049d30003, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T22:14:30,222 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41231-0x101409049d30002, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T22:14:30,222 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x101409049d30001, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T22:14:30,224 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39087-0x101409049d30000, quorum=127.0.0.1:60621, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-15T22:14:30,225 INFO [master/e611192d6313:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/e611192d6313,39087,1731708867142 from backup master directory 2024-11-15T22:14:30,436 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41231-0x101409049d30002, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T22:14:30,436 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43349-0x101409049d30003, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T22:14:30,436 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39087-0x101409049d30000, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/e611192d6313,39087,1731708867142 2024-11-15T22:14:30,436 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39087-0x101409049d30000, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T22:14:30,437 WARN [master/e611192d6313:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-15T22:14:30,437 INFO [master/e611192d6313:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=e611192d6313,39087,1731708867142 2024-11-15T22:14:30,437 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x101409049d30001, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T22:14:30,440 INFO [master/e611192d6313:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-15T22:14:30,443 INFO [master/e611192d6313:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-15T22:14:30,541 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/hbase.id] with ID: 1b4a912b-fe9b-4940-8b4d-31902347eb12 2024-11-15T22:14:30,541 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/.tmp/hbase.id 2024-11-15T22:14:30,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33631 is added to blk_1073741826_1002 (size=42) 2024-11-15T22:14:30,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39269 is added to blk_1073741826_1002 (size=42) 2024-11-15T22:14:30,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741826_1002 (size=42) 2024-11-15T22:14:30,967 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/.tmp/hbase.id]:[hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/hbase.id] 2024-11-15T22:14:31,025 INFO [master/e611192d6313:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T22:14:31,031 INFO [master/e611192d6313:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-15T22:14:31,053 INFO [master/e611192d6313:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 20ms. 2024-11-15T22:14:31,106 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43349-0x101409049d30003, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T22:14:31,106 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x101409049d30001, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T22:14:31,106 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41231-0x101409049d30002, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T22:14:31,106 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39087-0x101409049d30000, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T22:14:31,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741827_1003 (size=196) 2024-11-15T22:14:31,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33631 is added to blk_1073741827_1003 (size=196) 2024-11-15T22:14:31,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39269 is added to blk_1073741827_1003 (size=196) 2024-11-15T22:14:31,151 INFO [master/e611192d6313:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T22:14:31,153 INFO [master/e611192d6313:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-15T22:14:31,169 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:31,173 INFO [master/e611192d6313:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T22:14:31,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741828_1004 (size=1189) 2024-11-15T22:14:31,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33631 is added to blk_1073741828_1004 (size=1189) 2024-11-15T22:14:31,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39269 is added to blk_1073741828_1004 (size=1189) 2024-11-15T22:14:31,237 INFO [master/e611192d6313:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/MasterData/data/master/store 2024-11-15T22:14:31,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741829_1005 (size=34) 2024-11-15T22:14:31,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33631 is added to blk_1073741829_1005 (size=34) 2024-11-15T22:14:31,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39269 is added to blk_1073741829_1005 (size=34) 2024-11-15T22:14:31,288 INFO [master/e611192d6313:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-15T22:14:31,292 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:31,294 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-15T22:14:31,294 INFO [master/e611192d6313:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T22:14:31,294 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T22:14:31,297 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-15T22:14:31,297 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T22:14:31,297 INFO [master/e611192d6313:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T22:14:31,299 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731708871293Disabling compacts and flushes for region at 1731708871294 (+1 ms)Disabling writes for close at 1731708871297 (+3 ms)Writing region close event to WAL at 1731708871297Closed at 1731708871297 2024-11-15T22:14:31,306 WARN [master/e611192d6313:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/MasterData/data/master/store/.initializing 2024-11-15T22:14:31,306 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/MasterData/WALs/e611192d6313,39087,1731708867142 2024-11-15T22:14:31,318 INFO [master/e611192d6313:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T22:14:31,338 INFO [master/e611192d6313:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e611192d6313%2C39087%2C1731708867142, suffix=, logDir=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/MasterData/WALs/e611192d6313,39087,1731708867142, archiveDir=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/MasterData/oldWALs, maxLogs=10 2024-11-15T22:14:31,363 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/MasterData/WALs/e611192d6313,39087,1731708867142/e611192d6313%2C39087%2C1731708867142.1731708871343, exclude list is [], retry=0 2024-11-15T22:14:31,383 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39269,DS-b38187b6-2a6b-449a-a41a-28ace83252fc,DISK] 2024-11-15T22:14:31,384 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33631,DS-46524d7c-f751-41cf-9f4a-9f083a5e2adc,DISK] 2024-11-15T22:14:31,386 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-15T22:14:31,388 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40641,DS-0e1f6fcf-e559-4b2f-8ba2-2f7f76775d19,DISK] 2024-11-15T22:14:31,429 INFO [master/e611192d6313:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/MasterData/WALs/e611192d6313,39087,1731708867142/e611192d6313%2C39087%2C1731708867142.1731708871343 2024-11-15T22:14:31,431 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41859:41859),(127.0.0.1/127.0.0.1:34309:34309),(127.0.0.1/127.0.0.1:45219:45219)] 2024-11-15T22:14:31,431 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-15T22:14:31,432 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:31,435 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T22:14:31,436 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T22:14:31,476 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T22:14:31,506 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-15T22:14:31,510 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:31,513 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T22:14:31,514 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T22:14:31,518 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-15T22:14:31,518 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:31,519 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:14:31,520 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T22:14:31,523 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-15T22:14:31,523 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:31,524 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:14:31,524 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T22:14:31,527 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-15T22:14:31,528 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:31,529 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:14:31,529 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T22:14:31,534 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-15T22:14:31,535 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-15T22:14:31,541 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T22:14:31,542 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T22:14:31,546 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-15T22:14:31,550 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T22:14:31,555 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T22:14:31,556 INFO [master/e611192d6313:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65482641, jitterRate=-0.024232611060142517}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-15T22:14:31,564 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731708871451Initializing all the Stores at 1731708871453 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731708871453Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708871454 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708871454Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708871455 (+1 ms)Cleaning up temporary data from old regions at 1731708871542 (+87 ms)Region opened successfully at 1731708871564 (+22 ms) 2024-11-15T22:14:31,565 INFO [master/e611192d6313:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-15T22:14:31,605 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a2042e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e611192d6313/172.17.0.3:0 2024-11-15T22:14:31,645 INFO [master/e611192d6313:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-15T22:14:31,660 INFO [master/e611192d6313:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T22:14:31,661 INFO [master/e611192d6313:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T22:14:31,664 INFO [master/e611192d6313:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T22:14:31,668 INFO [master/e611192d6313:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 4 msec 2024-11-15T22:14:31,675 INFO [master/e611192d6313:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-15T22:14:31,675 INFO [master/e611192d6313:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-15T22:14:31,707 INFO [master/e611192d6313:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-15T22:14:31,720 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39087-0x101409049d30000, quorum=127.0.0.1:60621, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-15T22:14:31,758 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-15T22:14:31,764 INFO [master/e611192d6313:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-15T22:14:31,767 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39087-0x101409049d30000, quorum=127.0.0.1:60621, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-15T22:14:31,779 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-15T22:14:31,782 INFO [master/e611192d6313:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-15T22:14:31,785 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39087-0x101409049d30000, quorum=127.0.0.1:60621, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-15T22:14:31,795 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-15T22:14:31,796 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39087-0x101409049d30000, quorum=127.0.0.1:60621, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-15T22:14:31,805 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-15T22:14:31,832 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39087-0x101409049d30000, quorum=127.0.0.1:60621, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-15T22:14:31,842 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-15T22:14:31,858 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x101409049d30001, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-15T22:14:31,858 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39087-0x101409049d30000, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-15T22:14:31,858 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x101409049d30001, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T22:14:31,858 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39087-0x101409049d30000, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T22:14:31,858 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43349-0x101409049d30003, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-15T22:14:31,858 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43349-0x101409049d30003, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T22:14:31,859 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41231-0x101409049d30002, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-15T22:14:31,859 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41231-0x101409049d30002, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T22:14:31,863 INFO [master/e611192d6313:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=e611192d6313,39087,1731708867142, sessionid=0x101409049d30000, setting cluster-up flag (Was=false) 2024-11-15T22:14:31,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39087-0x101409049d30000, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T22:14:31,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41231-0x101409049d30002, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T22:14:31,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x101409049d30001, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T22:14:31,896 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43349-0x101409049d30003, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T22:14:31,927 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-15T22:14:31,929 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e611192d6313,39087,1731708867142 2024-11-15T22:14:32,042 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x101409049d30001, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T22:14:32,042 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41231-0x101409049d30002, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T22:14:32,042 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43349-0x101409049d30003, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T22:14:32,042 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39087-0x101409049d30000, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T22:14:32,074 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-15T22:14:32,075 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e611192d6313,39087,1731708867142 2024-11-15T22:14:32,082 INFO [master/e611192d6313:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-15T22:14:32,169 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T22:14:32,173 INFO [RS:0;e611192d6313:38729 {}] regionserver.HRegionServer(746): ClusterId : 1b4a912b-fe9b-4940-8b4d-31902347eb12 2024-11-15T22:14:32,173 INFO [RS:1;e611192d6313:41231 {}] regionserver.HRegionServer(746): ClusterId : 1b4a912b-fe9b-4940-8b4d-31902347eb12 2024-11-15T22:14:32,176 DEBUG [RS:0;e611192d6313:38729 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-15T22:14:32,176 DEBUG [RS:1;e611192d6313:41231 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-15T22:14:32,178 INFO [RS:2;e611192d6313:43349 {}] regionserver.HRegionServer(746): ClusterId : 1b4a912b-fe9b-4940-8b4d-31902347eb12 2024-11-15T22:14:32,178 DEBUG [RS:2;e611192d6313:43349 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-15T22:14:32,180 INFO [master/e611192d6313:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-15T22:14:32,188 INFO [master/e611192d6313:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-15T22:14:32,196 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: e611192d6313,39087,1731708867142 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-15T22:14:32,204 DEBUG [RS:2;e611192d6313:43349 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-15T22:14:32,204 DEBUG [RS:2;e611192d6313:43349 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-15T22:14:32,204 DEBUG [RS:1;e611192d6313:41231 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-15T22:14:32,204 DEBUG [RS:0;e611192d6313:38729 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-15T22:14:32,204 DEBUG [RS:1;e611192d6313:41231 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-15T22:14:32,204 DEBUG [RS:0;e611192d6313:38729 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-15T22:14:32,205 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/e611192d6313:0, corePoolSize=5, maxPoolSize=5 2024-11-15T22:14:32,205 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/e611192d6313:0, corePoolSize=5, maxPoolSize=5 2024-11-15T22:14:32,205 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/e611192d6313:0, corePoolSize=5, maxPoolSize=5 2024-11-15T22:14:32,205 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/e611192d6313:0, corePoolSize=5, maxPoolSize=5 2024-11-15T22:14:32,205 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/e611192d6313:0, corePoolSize=10, maxPoolSize=10 2024-11-15T22:14:32,206 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/e611192d6313:0, corePoolSize=1, maxPoolSize=1 2024-11-15T22:14:32,206 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/e611192d6313:0, corePoolSize=2, maxPoolSize=2 2024-11-15T22:14:32,206 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/e611192d6313:0, corePoolSize=1, maxPoolSize=1 2024-11-15T22:14:32,209 INFO [master/e611192d6313:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731708902209 2024-11-15T22:14:32,212 INFO [master/e611192d6313:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-15T22:14:32,213 INFO [master/e611192d6313:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-15T22:14:32,217 INFO [master/e611192d6313:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-15T22:14:32,218 INFO [master/e611192d6313:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-15T22:14:32,218 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-15T22:14:32,219 INFO [master/e611192d6313:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-15T22:14:32,219 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-15T22:14:32,219 INFO [master/e611192d6313:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-15T22:14:32,227 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:32,227 DEBUG [RS:2;e611192d6313:43349 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-15T22:14:32,227 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-15T22:14:32,228 DEBUG [RS:2;e611192d6313:43349 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@623a154e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e611192d6313/172.17.0.3:0 2024-11-15T22:14:32,229 DEBUG [RS:0;e611192d6313:38729 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-15T22:14:32,230 DEBUG [RS:0;e611192d6313:38729 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46baa4df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e611192d6313/172.17.0.3:0 2024-11-15T22:14:32,232 DEBUG [RS:1;e611192d6313:41231 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-15T22:14:32,232 DEBUG [RS:1;e611192d6313:41231 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a04b803, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e611192d6313/172.17.0.3:0 2024-11-15T22:14:32,230 INFO [master/e611192d6313:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-15T22:14:32,246 INFO [master/e611192d6313:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-15T22:14:32,248 INFO [master/e611192d6313:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-15T22:14:32,249 INFO [master/e611192d6313:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-15T22:14:32,253 DEBUG [RS:0;e611192d6313:38729 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;e611192d6313:38729 2024-11-15T22:14:32,257 INFO [RS:0;e611192d6313:38729 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-15T22:14:32,257 INFO [RS:0;e611192d6313:38729 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-15T22:14:32,257 DEBUG [RS:0;e611192d6313:38729 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-15T22:14:32,258 DEBUG [RS:2;e611192d6313:43349 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;e611192d6313:43349 2024-11-15T22:14:32,259 INFO [RS:2;e611192d6313:43349 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-15T22:14:32,259 INFO [RS:2;e611192d6313:43349 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-15T22:14:32,259 DEBUG [RS:2;e611192d6313:43349 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-15T22:14:32,260 INFO [master/e611192d6313:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-15T22:14:32,260 INFO [master/e611192d6313:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-15T22:14:32,260 INFO [RS:0;e611192d6313:38729 {}] regionserver.HRegionServer(2659): reportForDuty to master=e611192d6313,39087,1731708867142 with port=38729, startcode=1731708868111 2024-11-15T22:14:32,260 INFO [RS:2;e611192d6313:43349 {}] regionserver.HRegionServer(2659): reportForDuty to master=e611192d6313,39087,1731708867142 with port=43349, startcode=1731708869385 2024-11-15T22:14:32,264 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/e611192d6313:0:becomeActiveMaster-HFileCleaner.large.0-1731708872261,5,FailOnTimeoutGroup] 2024-11-15T22:14:32,265 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/e611192d6313:0:becomeActiveMaster-HFileCleaner.small.0-1731708872265,5,FailOnTimeoutGroup] 2024-11-15T22:14:32,266 INFO [master/e611192d6313:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-15T22:14:32,266 INFO [master/e611192d6313:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-15T22:14:32,266 DEBUG [RS:1;e611192d6313:41231 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;e611192d6313:41231 2024-11-15T22:14:32,267 INFO [RS:1;e611192d6313:41231 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-15T22:14:32,267 INFO [RS:1;e611192d6313:41231 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-15T22:14:32,267 DEBUG [RS:1;e611192d6313:41231 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-15T22:14:32,267 INFO [master/e611192d6313:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-15T22:14:32,267 INFO [master/e611192d6313:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-15T22:14:32,268 INFO [RS:1;e611192d6313:41231 {}] regionserver.HRegionServer(2659): reportForDuty to master=e611192d6313,39087,1731708867142 with port=41231, startcode=1731708869041 2024-11-15T22:14:32,275 DEBUG [RS:2;e611192d6313:43349 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-15T22:14:32,275 DEBUG [RS:0;e611192d6313:38729 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-15T22:14:32,276 DEBUG [RS:1;e611192d6313:41231 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-15T22:14:32,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741831_1007 (size=1321) 2024-11-15T22:14:32,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33631 is added to blk_1073741831_1007 (size=1321) 2024-11-15T22:14:32,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39269 is added to blk_1073741831_1007 (size=1321) 2024-11-15T22:14:32,286 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-15T22:14:32,286 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa 2024-11-15T22:14:32,349 INFO [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53325, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-15T22:14:32,352 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35301, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-15T22:14:32,353 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32981, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-15T22:14:32,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741832_1008 (size=32) 2024-11-15T22:14:32,362 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39087 {}] master.ServerManager(363): Checking decommissioned status of RegionServer e611192d6313,43349,1731708869385 2024-11-15T22:14:32,366 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39087 {}] master.ServerManager(517): Registering regionserver=e611192d6313,43349,1731708869385 2024-11-15T22:14:32,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39269 is added to blk_1073741832_1008 (size=32) 2024-11-15T22:14:32,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33631 is added to blk_1073741832_1008 (size=32) 2024-11-15T22:14:32,375 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:32,379 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-15T22:14:32,383 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-15T22:14:32,384 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:32,386 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T22:14:32,386 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-15T22:14:32,387 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39087 {}] master.ServerManager(363): Checking decommissioned status of RegionServer e611192d6313,41231,1731708869041 2024-11-15T22:14:32,388 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39087 {}] master.ServerManager(517): Registering regionserver=e611192d6313,41231,1731708869041 2024-11-15T22:14:32,391 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-15T22:14:32,391 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:32,393 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T22:14:32,396 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-15T22:14:32,397 DEBUG [RS:2;e611192d6313:43349 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa 2024-11-15T22:14:32,397 DEBUG [RS:2;e611192d6313:43349 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35755 2024-11-15T22:14:32,397 DEBUG [RS:2;e611192d6313:43349 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-15T22:14:32,403 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39087 {}] master.ServerManager(363): Checking decommissioned status of RegionServer e611192d6313,38729,1731708868111 2024-11-15T22:14:32,403 DEBUG [RS:1;e611192d6313:41231 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa 2024-11-15T22:14:32,403 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39087 {}] master.ServerManager(517): Registering regionserver=e611192d6313,38729,1731708868111 2024-11-15T22:14:32,403 DEBUG [RS:1;e611192d6313:41231 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35755 2024-11-15T22:14:32,403 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-15T22:14:32,403 DEBUG [RS:1;e611192d6313:41231 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-15T22:14:32,403 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:32,405 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T22:14:32,405 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-15T22:14:32,410 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-15T22:14:32,410 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:32,412 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T22:14:32,413 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-15T22:14:32,415 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/hbase/meta/1588230740 2024-11-15T22:14:32,416 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/hbase/meta/1588230740 2024-11-15T22:14:32,417 DEBUG [RS:0;e611192d6313:38729 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa 2024-11-15T22:14:32,417 DEBUG [RS:0;e611192d6313:38729 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35755 2024-11-15T22:14:32,417 DEBUG [RS:0;e611192d6313:38729 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-15T22:14:32,431 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-15T22:14:32,431 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-15T22:14:32,432 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-15T22:14:32,436 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-15T22:14:32,444 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T22:14:32,445 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61373630, jitterRate=-0.08546164631843567}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-15T22:14:32,450 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731708872375Initializing all the Stores at 1731708872378 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731708872378Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731708872378Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708872378Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731708872378Cleaning up temporary data from old regions at 1731708872431 (+53 ms)Region opened successfully at 1731708872450 (+19 ms) 2024-11-15T22:14:32,450 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-15T22:14:32,450 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-15T22:14:32,450 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-15T22:14:32,450 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-15T22:14:32,450 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-15T22:14:32,452 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-15T22:14:32,452 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731708872450Disabling compacts and flushes for region at 1731708872450Disabling writes for close at 1731708872450Writing region close event to WAL at 1731708872451 (+1 ms)Closed at 1731708872452 (+1 ms) 2024-11-15T22:14:32,456 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39087-0x101409049d30000, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-15T22:14:32,458 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-15T22:14:32,458 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-15T22:14:32,466 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-15T22:14:32,474 DEBUG [RS:2;e611192d6313:43349 {}] zookeeper.ZKUtil(111): regionserver:43349-0x101409049d30003, quorum=127.0.0.1:60621, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/e611192d6313,43349,1731708869385 2024-11-15T22:14:32,474 WARN [RS:2;e611192d6313:43349 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-15T22:14:32,475 DEBUG [RS:0;e611192d6313:38729 {}] zookeeper.ZKUtil(111): regionserver:38729-0x101409049d30001, quorum=127.0.0.1:60621, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/e611192d6313,38729,1731708868111 2024-11-15T22:14:32,475 INFO [RS:2;e611192d6313:43349 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T22:14:32,475 WARN [RS:0;e611192d6313:38729 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-15T22:14:32,475 INFO [RS:0;e611192d6313:38729 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T22:14:32,475 DEBUG [RS:2;e611192d6313:43349 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/WALs/e611192d6313,43349,1731708869385 2024-11-15T22:14:32,475 DEBUG [RS:0;e611192d6313:38729 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/WALs/e611192d6313,38729,1731708868111 2024-11-15T22:14:32,477 DEBUG [RS:1;e611192d6313:41231 {}] zookeeper.ZKUtil(111): regionserver:41231-0x101409049d30002, quorum=127.0.0.1:60621, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/e611192d6313,41231,1731708869041 2024-11-15T22:14:32,477 WARN [RS:1;e611192d6313:41231 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-15T22:14:32,477 INFO [RS:1;e611192d6313:41231 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T22:14:32,477 DEBUG [RS:1;e611192d6313:41231 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/WALs/e611192d6313,41231,1731708869041 2024-11-15T22:14:32,479 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [e611192d6313,43349,1731708869385] 2024-11-15T22:14:32,479 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [e611192d6313,38729,1731708868111] 2024-11-15T22:14:32,479 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [e611192d6313,41231,1731708869041] 2024-11-15T22:14:32,484 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T22:14:32,490 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-15T22:14:32,514 INFO [RS:1;e611192d6313:41231 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-15T22:14:32,514 INFO [RS:0;e611192d6313:38729 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-15T22:14:32,514 INFO [RS:2;e611192d6313:43349 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-15T22:14:32,533 INFO [RS:0;e611192d6313:38729 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-15T22:14:32,533 INFO [RS:1;e611192d6313:41231 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-15T22:14:32,534 INFO [RS:2;e611192d6313:43349 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-15T22:14:32,541 INFO [RS:0;e611192d6313:38729 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-15T22:14:32,541 INFO [RS:2;e611192d6313:43349 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-15T22:14:32,541 INFO [RS:1;e611192d6313:41231 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-15T22:14:32,541 INFO [RS:1;e611192d6313:41231 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T22:14:32,541 INFO [RS:2;e611192d6313:43349 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T22:14:32,541 INFO [RS:0;e611192d6313:38729 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T22:14:32,542 INFO [RS:2;e611192d6313:43349 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-15T22:14:32,542 INFO [RS:0;e611192d6313:38729 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-15T22:14:32,543 INFO [RS:1;e611192d6313:41231 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-15T22:14:32,549 INFO [RS:0;e611192d6313:38729 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-15T22:14:32,551 INFO [RS:0;e611192d6313:38729 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-15T22:14:32,551 DEBUG [RS:0;e611192d6313:38729 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/e611192d6313:0, corePoolSize=1, maxPoolSize=1 2024-11-15T22:14:32,552 DEBUG [RS:0;e611192d6313:38729 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/e611192d6313:0, corePoolSize=1, maxPoolSize=1 2024-11-15T22:14:32,552 DEBUG [RS:0;e611192d6313:38729 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/e611192d6313:0, corePoolSize=1, maxPoolSize=1 2024-11-15T22:14:32,552 DEBUG [RS:0;e611192d6313:38729 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/e611192d6313:0, corePoolSize=1, maxPoolSize=1 2024-11-15T22:14:32,552 DEBUG [RS:0;e611192d6313:38729 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/e611192d6313:0, corePoolSize=1, maxPoolSize=1 2024-11-15T22:14:32,552 INFO [RS:2;e611192d6313:43349 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-15T22:14:32,552 DEBUG [RS:0;e611192d6313:38729 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/e611192d6313:0, corePoolSize=2, maxPoolSize=2 2024-11-15T22:14:32,552 INFO [RS:2;e611192d6313:43349 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-15T22:14:32,552 DEBUG [RS:0;e611192d6313:38729 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/e611192d6313:0, corePoolSize=1, maxPoolSize=1 2024-11-15T22:14:32,553 DEBUG [RS:0;e611192d6313:38729 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/e611192d6313:0, corePoolSize=1, maxPoolSize=1 2024-11-15T22:14:32,553 DEBUG [RS:2;e611192d6313:43349 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/e611192d6313:0, corePoolSize=1, maxPoolSize=1 2024-11-15T22:14:32,553 INFO [RS:1;e611192d6313:41231 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-15T22:14:32,553 DEBUG [RS:0;e611192d6313:38729 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/e611192d6313:0, corePoolSize=1, maxPoolSize=1 2024-11-15T22:14:32,553 DEBUG [RS:2;e611192d6313:43349 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/e611192d6313:0, corePoolSize=1, maxPoolSize=1 2024-11-15T22:14:32,553 DEBUG [RS:0;e611192d6313:38729 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/e611192d6313:0, corePoolSize=1, maxPoolSize=1 2024-11-15T22:14:32,553 INFO [RS:1;e611192d6313:41231 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-15T22:14:32,553 DEBUG [RS:2;e611192d6313:43349 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/e611192d6313:0, corePoolSize=1, maxPoolSize=1 2024-11-15T22:14:32,553 DEBUG [RS:0;e611192d6313:38729 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/e611192d6313:0, corePoolSize=1, maxPoolSize=1 2024-11-15T22:14:32,553 DEBUG [RS:1;e611192d6313:41231 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/e611192d6313:0, corePoolSize=1, maxPoolSize=1 2024-11-15T22:14:32,553 DEBUG [RS:2;e611192d6313:43349 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/e611192d6313:0, corePoolSize=1, maxPoolSize=1 2024-11-15T22:14:32,553 DEBUG [RS:0;e611192d6313:38729 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/e611192d6313:0, corePoolSize=1, maxPoolSize=1 2024-11-15T22:14:32,553 DEBUG [RS:1;e611192d6313:41231 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/e611192d6313:0, corePoolSize=1, maxPoolSize=1 2024-11-15T22:14:32,553 DEBUG [RS:0;e611192d6313:38729 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/e611192d6313:0, corePoolSize=3, maxPoolSize=3 2024-11-15T22:14:32,553 DEBUG [RS:2;e611192d6313:43349 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/e611192d6313:0, corePoolSize=1, maxPoolSize=1 2024-11-15T22:14:32,553 DEBUG [RS:1;e611192d6313:41231 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/e611192d6313:0, corePoolSize=1, maxPoolSize=1 2024-11-15T22:14:32,553 DEBUG [RS:0;e611192d6313:38729 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0, corePoolSize=3, maxPoolSize=3 2024-11-15T22:14:32,553 DEBUG [RS:2;e611192d6313:43349 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/e611192d6313:0, corePoolSize=2, maxPoolSize=2 2024-11-15T22:14:32,553 DEBUG [RS:1;e611192d6313:41231 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/e611192d6313:0, corePoolSize=1, maxPoolSize=1 2024-11-15T22:14:32,554 DEBUG [RS:1;e611192d6313:41231 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/e611192d6313:0, corePoolSize=1, maxPoolSize=1 2024-11-15T22:14:32,554 DEBUG [RS:2;e611192d6313:43349 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/e611192d6313:0, corePoolSize=1, maxPoolSize=1 2024-11-15T22:14:32,554 DEBUG [RS:1;e611192d6313:41231 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/e611192d6313:0, corePoolSize=2, maxPoolSize=2 2024-11-15T22:14:32,554 DEBUG [RS:2;e611192d6313:43349 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/e611192d6313:0, corePoolSize=1, maxPoolSize=1 2024-11-15T22:14:32,554 DEBUG [RS:1;e611192d6313:41231 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/e611192d6313:0, corePoolSize=1, maxPoolSize=1 2024-11-15T22:14:32,554 DEBUG [RS:2;e611192d6313:43349 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/e611192d6313:0, corePoolSize=1, maxPoolSize=1 2024-11-15T22:14:32,554 DEBUG [RS:1;e611192d6313:41231 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/e611192d6313:0, corePoolSize=1, maxPoolSize=1 2024-11-15T22:14:32,554 DEBUG [RS:2;e611192d6313:43349 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/e611192d6313:0, corePoolSize=1, maxPoolSize=1 2024-11-15T22:14:32,554 DEBUG [RS:1;e611192d6313:41231 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/e611192d6313:0, corePoolSize=1, maxPoolSize=1 2024-11-15T22:14:32,554 DEBUG [RS:2;e611192d6313:43349 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/e611192d6313:0, corePoolSize=1, maxPoolSize=1 2024-11-15T22:14:32,554 DEBUG [RS:1;e611192d6313:41231 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/e611192d6313:0, corePoolSize=1, maxPoolSize=1 2024-11-15T22:14:32,554 DEBUG [RS:1;e611192d6313:41231 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/e611192d6313:0, corePoolSize=1, maxPoolSize=1 2024-11-15T22:14:32,554 DEBUG [RS:2;e611192d6313:43349 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/e611192d6313:0, corePoolSize=1, maxPoolSize=1 2024-11-15T22:14:32,554 DEBUG [RS:1;e611192d6313:41231 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/e611192d6313:0, corePoolSize=1, maxPoolSize=1 2024-11-15T22:14:32,555 DEBUG [RS:2;e611192d6313:43349 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/e611192d6313:0, corePoolSize=3, maxPoolSize=3 2024-11-15T22:14:32,555 DEBUG [RS:1;e611192d6313:41231 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/e611192d6313:0, corePoolSize=3, maxPoolSize=3 2024-11-15T22:14:32,555 DEBUG [RS:2;e611192d6313:43349 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0, corePoolSize=3, maxPoolSize=3 2024-11-15T22:14:32,555 DEBUG [RS:1;e611192d6313:41231 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/e611192d6313:0, corePoolSize=3, maxPoolSize=3 2024-11-15T22:14:32,561 INFO [RS:2;e611192d6313:43349 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T22:14:32,561 INFO [RS:1;e611192d6313:41231 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T22:14:32,561 INFO [RS:0;e611192d6313:38729 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T22:14:32,561 INFO [RS:2;e611192d6313:43349 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T22:14:32,561 INFO [RS:1;e611192d6313:41231 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T22:14:32,561 INFO [RS:0;e611192d6313:38729 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T22:14:32,561 INFO [RS:2;e611192d6313:43349 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T22:14:32,561 INFO [RS:1;e611192d6313:41231 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T22:14:32,561 INFO [RS:0;e611192d6313:38729 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T22:14:32,561 INFO [RS:2;e611192d6313:43349 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-15T22:14:32,561 INFO [RS:1;e611192d6313:41231 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-15T22:14:32,562 INFO [RS:0;e611192d6313:38729 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-15T22:14:32,562 INFO [RS:2;e611192d6313:43349 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-15T22:14:32,562 INFO [RS:1;e611192d6313:41231 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-15T22:14:32,562 INFO [RS:0;e611192d6313:38729 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-15T22:14:32,562 INFO [RS:2;e611192d6313:43349 {}] hbase.ChoreService(168): Chore ScheduledChore name=e611192d6313,43349,1731708869385-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-15T22:14:32,562 INFO [RS:1;e611192d6313:41231 {}] hbase.ChoreService(168): Chore ScheduledChore name=e611192d6313,41231,1731708869041-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-15T22:14:32,562 INFO [RS:0;e611192d6313:38729 {}] hbase.ChoreService(168): Chore ScheduledChore name=e611192d6313,38729,1731708868111-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-15T22:14:32,587 INFO [RS:2;e611192d6313:43349 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-15T22:14:32,590 INFO [RS:0;e611192d6313:38729 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-15T22:14:32,590 INFO [RS:2;e611192d6313:43349 {}] hbase.ChoreService(168): Chore ScheduledChore name=e611192d6313,43349,1731708869385-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T22:14:32,590 INFO [RS:0;e611192d6313:38729 {}] hbase.ChoreService(168): Chore ScheduledChore name=e611192d6313,38729,1731708868111-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T22:14:32,590 INFO [RS:1;e611192d6313:41231 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-15T22:14:32,590 INFO [RS:0;e611192d6313:38729 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T22:14:32,590 INFO [RS:1;e611192d6313:41231 {}] hbase.ChoreService(168): Chore ScheduledChore name=e611192d6313,41231,1731708869041-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T22:14:32,590 INFO [RS:0;e611192d6313:38729 {}] regionserver.Replication(171): e611192d6313,38729,1731708868111 started 2024-11-15T22:14:32,590 INFO [RS:1;e611192d6313:41231 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T22:14:32,591 INFO [RS:1;e611192d6313:41231 {}] regionserver.Replication(171): e611192d6313,41231,1731708869041 started 2024-11-15T22:14:32,592 INFO [RS:2;e611192d6313:43349 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T22:14:32,592 INFO [RS:2;e611192d6313:43349 {}] regionserver.Replication(171): e611192d6313,43349,1731708869385 started 2024-11-15T22:14:32,610 INFO [RS:2;e611192d6313:43349 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T22:14:32,610 INFO [RS:2;e611192d6313:43349 {}] regionserver.HRegionServer(1482): Serving as e611192d6313,43349,1731708869385, RpcServer on e611192d6313/172.17.0.3:43349, sessionid=0x101409049d30003 2024-11-15T22:14:32,611 DEBUG [RS:2;e611192d6313:43349 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-15T22:14:32,611 DEBUG [RS:2;e611192d6313:43349 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager e611192d6313,43349,1731708869385 2024-11-15T22:14:32,612 DEBUG [RS:2;e611192d6313:43349 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e611192d6313,43349,1731708869385' 2024-11-15T22:14:32,612 DEBUG [RS:2;e611192d6313:43349 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-15T22:14:32,613 DEBUG [RS:2;e611192d6313:43349 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-15T22:14:32,614 DEBUG [RS:2;e611192d6313:43349 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-15T22:14:32,614 DEBUG [RS:2;e611192d6313:43349 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-15T22:14:32,614 DEBUG [RS:2;e611192d6313:43349 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager e611192d6313,43349,1731708869385 2024-11-15T22:14:32,614 DEBUG [RS:2;e611192d6313:43349 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e611192d6313,43349,1731708869385' 2024-11-15T22:14:32,614 DEBUG [RS:2;e611192d6313:43349 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-15T22:14:32,615 DEBUG [RS:2;e611192d6313:43349 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-15T22:14:32,615 DEBUG [RS:2;e611192d6313:43349 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-15T22:14:32,615 INFO [RS:2;e611192d6313:43349 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-15T22:14:32,615 INFO [RS:2;e611192d6313:43349 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-15T22:14:32,619 INFO [RS:1;e611192d6313:41231 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T22:14:32,619 INFO [RS:1;e611192d6313:41231 {}] regionserver.HRegionServer(1482): Serving as e611192d6313,41231,1731708869041, RpcServer on e611192d6313/172.17.0.3:41231, sessionid=0x101409049d30002 2024-11-15T22:14:32,619 DEBUG [RS:1;e611192d6313:41231 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-15T22:14:32,620 DEBUG [RS:1;e611192d6313:41231 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager e611192d6313,41231,1731708869041 2024-11-15T22:14:32,620 DEBUG [RS:1;e611192d6313:41231 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e611192d6313,41231,1731708869041' 2024-11-15T22:14:32,620 DEBUG [RS:1;e611192d6313:41231 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-15T22:14:32,621 DEBUG [RS:1;e611192d6313:41231 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-15T22:14:32,621 DEBUG [RS:1;e611192d6313:41231 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-15T22:14:32,621 DEBUG [RS:1;e611192d6313:41231 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-15T22:14:32,621 DEBUG [RS:1;e611192d6313:41231 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager e611192d6313,41231,1731708869041 2024-11-15T22:14:32,622 DEBUG [RS:1;e611192d6313:41231 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e611192d6313,41231,1731708869041' 2024-11-15T22:14:32,622 DEBUG [RS:1;e611192d6313:41231 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-15T22:14:32,622 DEBUG [RS:1;e611192d6313:41231 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-15T22:14:32,623 DEBUG [RS:1;e611192d6313:41231 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-15T22:14:32,623 INFO [RS:1;e611192d6313:41231 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-15T22:14:32,623 INFO [RS:1;e611192d6313:41231 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-15T22:14:32,625 INFO [RS:0;e611192d6313:38729 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T22:14:32,625 INFO [RS:0;e611192d6313:38729 {}] regionserver.HRegionServer(1482): Serving as e611192d6313,38729,1731708868111, RpcServer on e611192d6313/172.17.0.3:38729, sessionid=0x101409049d30001 2024-11-15T22:14:32,625 DEBUG [RS:0;e611192d6313:38729 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-15T22:14:32,626 DEBUG [RS:0;e611192d6313:38729 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager e611192d6313,38729,1731708868111 2024-11-15T22:14:32,626 DEBUG [RS:0;e611192d6313:38729 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e611192d6313,38729,1731708868111' 2024-11-15T22:14:32,626 DEBUG [RS:0;e611192d6313:38729 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-15T22:14:32,627 DEBUG [RS:0;e611192d6313:38729 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-15T22:14:32,628 DEBUG [RS:0;e611192d6313:38729 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-15T22:14:32,628 DEBUG [RS:0;e611192d6313:38729 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-15T22:14:32,628 DEBUG [RS:0;e611192d6313:38729 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager e611192d6313,38729,1731708868111 2024-11-15T22:14:32,628 DEBUG [RS:0;e611192d6313:38729 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e611192d6313,38729,1731708868111' 2024-11-15T22:14:32,628 DEBUG [RS:0;e611192d6313:38729 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-15T22:14:32,628 DEBUG [RS:0;e611192d6313:38729 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-15T22:14:32,629 DEBUG [RS:0;e611192d6313:38729 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-15T22:14:32,629 INFO [RS:0;e611192d6313:38729 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-15T22:14:32,629 INFO [RS:0;e611192d6313:38729 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-15T22:14:32,641 WARN [e611192d6313:39087 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-15T22:14:32,722 INFO [RS:2;e611192d6313:43349 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T22:14:32,724 INFO [RS:1;e611192d6313:41231 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T22:14:32,725 INFO [RS:2;e611192d6313:43349 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e611192d6313%2C43349%2C1731708869385, suffix=, logDir=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/WALs/e611192d6313,43349,1731708869385, archiveDir=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/oldWALs, maxLogs=32 2024-11-15T22:14:32,728 INFO [RS:1;e611192d6313:41231 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e611192d6313%2C41231%2C1731708869041, suffix=, logDir=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/WALs/e611192d6313,41231,1731708869041, archiveDir=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/oldWALs, maxLogs=32 2024-11-15T22:14:32,730 INFO [RS:0;e611192d6313:38729 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T22:14:32,741 INFO [RS:0;e611192d6313:38729 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e611192d6313%2C38729%2C1731708868111, suffix=, logDir=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/WALs/e611192d6313,38729,1731708868111, archiveDir=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/oldWALs, maxLogs=32 2024-11-15T22:14:32,756 DEBUG [RS:2;e611192d6313:43349 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/WALs/e611192d6313,43349,1731708869385/e611192d6313%2C43349%2C1731708869385.1731708872733, exclude list is [], retry=0 2024-11-15T22:14:32,770 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40641,DS-0e1f6fcf-e559-4b2f-8ba2-2f7f76775d19,DISK] 2024-11-15T22:14:32,771 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33631,DS-46524d7c-f751-41cf-9f4a-9f083a5e2adc,DISK] 2024-11-15T22:14:32,772 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39269,DS-b38187b6-2a6b-449a-a41a-28ace83252fc,DISK] 2024-11-15T22:14:32,776 DEBUG [RS:0;e611192d6313:38729 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/WALs/e611192d6313,38729,1731708868111/e611192d6313%2C38729%2C1731708868111.1731708872744, exclude list is [], retry=0 2024-11-15T22:14:32,776 DEBUG [RS:1;e611192d6313:41231 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/WALs/e611192d6313,41231,1731708869041/e611192d6313%2C41231%2C1731708869041.1731708872733, exclude list is [], retry=0 2024-11-15T22:14:32,786 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40641,DS-0e1f6fcf-e559-4b2f-8ba2-2f7f76775d19,DISK] 2024-11-15T22:14:32,787 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33631,DS-46524d7c-f751-41cf-9f4a-9f083a5e2adc,DISK] 2024-11-15T22:14:32,787 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39269,DS-b38187b6-2a6b-449a-a41a-28ace83252fc,DISK] 2024-11-15T22:14:32,789 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33631,DS-46524d7c-f751-41cf-9f4a-9f083a5e2adc,DISK] 2024-11-15T22:14:32,789 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39269,DS-b38187b6-2a6b-449a-a41a-28ace83252fc,DISK] 2024-11-15T22:14:32,789 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40641,DS-0e1f6fcf-e559-4b2f-8ba2-2f7f76775d19,DISK] 2024-11-15T22:14:32,812 INFO [RS:2;e611192d6313:43349 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/WALs/e611192d6313,43349,1731708869385/e611192d6313%2C43349%2C1731708869385.1731708872733 2024-11-15T22:14:32,829 DEBUG [RS:2;e611192d6313:43349 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45219:45219),(127.0.0.1/127.0.0.1:34309:34309),(127.0.0.1/127.0.0.1:41859:41859)] 2024-11-15T22:14:32,838 INFO [RS:0;e611192d6313:38729 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/WALs/e611192d6313,38729,1731708868111/e611192d6313%2C38729%2C1731708868111.1731708872744 2024-11-15T22:14:32,842 DEBUG [RS:0;e611192d6313:38729 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34309:34309),(127.0.0.1/127.0.0.1:45219:45219),(127.0.0.1/127.0.0.1:41859:41859)] 2024-11-15T22:14:32,850 INFO [RS:1;e611192d6313:41231 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/WALs/e611192d6313,41231,1731708869041/e611192d6313%2C41231%2C1731708869041.1731708872733 2024-11-15T22:14:32,853 DEBUG [RS:1;e611192d6313:41231 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41859:41859),(127.0.0.1/127.0.0.1:45219:45219),(127.0.0.1/127.0.0.1:34309:34309)] 2024-11-15T22:14:32,894 DEBUG [e611192d6313:39087 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-15T22:14:32,904 DEBUG [e611192d6313:39087 {}] balancer.BalancerClusterState(204): Hosts are {e611192d6313=0} racks are {/default-rack=0} 2024-11-15T22:14:32,911 DEBUG [e611192d6313:39087 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-15T22:14:32,912 DEBUG [e611192d6313:39087 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-15T22:14:32,912 DEBUG [e611192d6313:39087 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-15T22:14:32,912 DEBUG [e611192d6313:39087 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-15T22:14:32,912 DEBUG [e611192d6313:39087 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-15T22:14:32,912 DEBUG [e611192d6313:39087 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-15T22:14:32,912 INFO [e611192d6313:39087 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-15T22:14:32,912 INFO [e611192d6313:39087 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-15T22:14:32,912 INFO [e611192d6313:39087 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-15T22:14:32,912 DEBUG [e611192d6313:39087 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-15T22:14:32,920 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=e611192d6313,43349,1731708869385 2024-11-15T22:14:32,926 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e611192d6313,43349,1731708869385, state=OPENING 2024-11-15T22:14:33,000 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-15T22:14:33,011 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x101409049d30001, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T22:14:33,011 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43349-0x101409049d30003, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T22:14:33,011 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41231-0x101409049d30002, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T22:14:33,011 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39087-0x101409049d30000, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T22:14:33,012 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:33,012 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:33,013 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:33,013 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:33,016 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T22:14:33,018 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=e611192d6313,43349,1731708869385}] 2024-11-15T22:14:33,204 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-15T22:14:33,206 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55547, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-15T22:14:33,219 INFO [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-15T22:14:33,219 INFO [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T22:14:33,220 INFO [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-15T22:14:33,223 INFO [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e611192d6313%2C43349%2C1731708869385.meta, suffix=.meta, logDir=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/WALs/e611192d6313,43349,1731708869385, archiveDir=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/oldWALs, maxLogs=32 2024-11-15T22:14:33,239 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/WALs/e611192d6313,43349,1731708869385/e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, exclude list is [], retry=0 2024-11-15T22:14:33,244 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40641,DS-0e1f6fcf-e559-4b2f-8ba2-2f7f76775d19,DISK] 2024-11-15T22:14:33,244 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39269,DS-b38187b6-2a6b-449a-a41a-28ace83252fc,DISK] 2024-11-15T22:14:33,244 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33631,DS-46524d7c-f751-41cf-9f4a-9f083a5e2adc,DISK] 2024-11-15T22:14:33,254 INFO [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/WALs/e611192d6313,43349,1731708869385/e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta 2024-11-15T22:14:33,254 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34309:34309),(127.0.0.1/127.0.0.1:41859:41859),(127.0.0.1/127.0.0.1:45219:45219)] 2024-11-15T22:14:33,255 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-15T22:14:33,256 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-15T22:14:33,259 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-15T22:14:33,263 INFO [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-15T22:14:33,267 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-15T22:14:33,268 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:33,268 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-15T22:14:33,268 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-15T22:14:33,275 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-15T22:14:33,277 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-15T22:14:33,277 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:33,278 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T22:14:33,278 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-15T22:14:33,280 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-15T22:14:33,281 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:33,282 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T22:14:33,282 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-15T22:14:33,284 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-15T22:14:33,284 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:33,285 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T22:14:33,286 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-15T22:14:33,287 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-15T22:14:33,287 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:33,288 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T22:14:33,289 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-15T22:14:33,291 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/hbase/meta/1588230740 2024-11-15T22:14:33,294 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/hbase/meta/1588230740 2024-11-15T22:14:33,296 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-15T22:14:33,296 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-15T22:14:33,297 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-15T22:14:33,300 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-15T22:14:33,302 INFO [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60311149, jitterRate=-0.10129384696483612}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-15T22:14:33,302 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-15T22:14:33,304 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731708873269Writing region info on filesystem at 1731708873269Initializing all the Stores at 1731708873271 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731708873272 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731708873274 (+2 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708873274Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731708873274Cleaning up temporary data from old regions at 1731708873296 (+22 ms)Running coprocessor post-open hooks at 1731708873303 (+7 ms)Region opened successfully at 1731708873304 (+1 ms) 2024-11-15T22:14:33,312 INFO [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731708873195 2024-11-15T22:14:33,324 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-15T22:14:33,325 INFO [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-15T22:14:33,327 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=e611192d6313,43349,1731708869385 2024-11-15T22:14:33,329 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e611192d6313,43349,1731708869385, state=OPEN 2024-11-15T22:14:33,342 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x101409049d30001, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:33,342 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41231-0x101409049d30002, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:33,342 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39087-0x101409049d30000, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:33,342 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43349-0x101409049d30003, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:33,342 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:33,342 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:33,342 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:33,342 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:33,343 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=e611192d6313,43349,1731708869385 2024-11-15T22:14:33,350 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-15T22:14:33,350 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e611192d6313,43349,1731708869385 in 325 msec 2024-11-15T22:14:33,371 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-15T22:14:33,371 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 887 msec 2024-11-15T22:14:33,374 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-15T22:14:33,374 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-15T22:14:33,398 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:14:33,400 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,43349,1731708869385, seqNum=-1] 2024-11-15T22:14:33,421 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:14:33,423 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53667, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:14:33,454 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.3270 sec 2024-11-15T22:14:33,454 INFO [master/e611192d6313:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731708873454, completionTime=-1 2024-11-15T22:14:33,457 INFO [master/e611192d6313:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-15T22:14:33,458 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-15T22:14:33,490 INFO [master/e611192d6313:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=3 2024-11-15T22:14:33,490 INFO [master/e611192d6313:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731708933490 2024-11-15T22:14:33,490 INFO [master/e611192d6313:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731708993490 2024-11-15T22:14:33,491 INFO [master/e611192d6313:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 32 msec 2024-11-15T22:14:33,493 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-15T22:14:33,501 INFO [master/e611192d6313:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e611192d6313,39087,1731708867142-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T22:14:33,501 INFO [master/e611192d6313:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e611192d6313,39087,1731708867142-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T22:14:33,501 INFO [master/e611192d6313:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e611192d6313,39087,1731708867142-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T22:14:33,503 INFO [master/e611192d6313:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-e611192d6313:39087, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T22:14:33,503 INFO [master/e611192d6313:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-15T22:14:33,504 INFO [master/e611192d6313:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-15T22:14:33,510 DEBUG [master/e611192d6313:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-15T22:14:33,537 INFO [master/e611192d6313:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.100sec 2024-11-15T22:14:33,539 INFO [master/e611192d6313:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-15T22:14:33,541 INFO [master/e611192d6313:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-15T22:14:33,542 INFO [master/e611192d6313:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-15T22:14:33,543 INFO [master/e611192d6313:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-15T22:14:33,543 INFO [master/e611192d6313:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-15T22:14:33,544 INFO [master/e611192d6313:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e611192d6313,39087,1731708867142-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-15T22:14:33,544 INFO [master/e611192d6313:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e611192d6313,39087,1731708867142-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-15T22:14:33,550 DEBUG [master/e611192d6313:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-15T22:14:33,551 INFO [master/e611192d6313:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-15T22:14:33,552 INFO [master/e611192d6313:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e611192d6313,39087,1731708867142-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T22:14:33,582 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41bb0b4a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:14:33,587 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-15T22:14:33,587 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-15T22:14:33,591 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e611192d6313,39087,-1 for getting cluster id 2024-11-15T22:14:33,594 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T22:14:33,604 DEBUG [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1b4a912b-fe9b-4940-8b4d-31902347eb12' 2024-11-15T22:14:33,607 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T22:14:33,607 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1b4a912b-fe9b-4940-8b4d-31902347eb12" 2024-11-15T22:14:33,608 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@140a9834, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:14:33,608 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e611192d6313,39087,-1] 2024-11-15T22:14:33,611 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T22:14:33,613 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:14:33,615 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57324, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T22:14:33,618 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@109acbb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T22:14:33,618 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:14:33,627 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,43349,1731708869385, seqNum=-1] 2024-11-15T22:14:33,628 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:14:33,631 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44836, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:14:33,657 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=e611192d6313,39087,1731708867142 2024-11-15T22:14:33,662 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T22:14:33,668 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is e611192d6313,39087,1731708867142 2024-11-15T22:14:33,671 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5ac98032 2024-11-15T22:14:33,673 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T22:14:33,679 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57340, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T22:14:33,688 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39087 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'test', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T22:14:33,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39087 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test 2024-11-15T22:14:33,713 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T22:14:33,715 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39087 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "test" procId is: 4 2024-11-15T22:14:33,716 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:33,721 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T22:14:33,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T22:14:33,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39269 is added to blk_1073741837_1013 (size=358) 2024-11-15T22:14:33,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33631 is added to blk_1073741837_1013 (size=358) 2024-11-15T22:14:33,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741837_1013 (size=358) 2024-11-15T22:14:33,851 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f2783381a48c13c2f75c07e5029903cb, NAME => 'test,,1731708873681.f2783381a48c13c2f75c07e5029903cb.', STARTKEY => '', ENDKEY => 'bbb'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa 2024-11-15T22:14:33,852 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 2424ff4b6d907aafacef6f661d88ef95, NAME => 'test,ccc,1731708873681.2424ff4b6d907aafacef6f661d88ef95.', STARTKEY => 'ccc', ENDKEY => 'ddd'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa 2024-11-15T22:14:33,852 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 897a7bb57a002d365489ce583eb7f84f, NAME => 'test,bbb,1731708873681.897a7bb57a002d365489ce583eb7f84f.', STARTKEY => 'bbb', ENDKEY => 'ccc'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa 2024-11-15T22:14:33,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741838_1014 (size=42) 2024-11-15T22:14:33,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39269 is added to blk_1073741838_1014 (size=42) 2024-11-15T22:14:33,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33631 is added to blk_1073741838_1014 (size=42) 2024-11-15T22:14:33,886 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,,1731708873681.f2783381a48c13c2f75c07e5029903cb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:33,886 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing f2783381a48c13c2f75c07e5029903cb, disabling compactions & flushes 2024-11-15T22:14:33,886 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,,1731708873681.f2783381a48c13c2f75c07e5029903cb. 2024-11-15T22:14:33,886 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1731708873681.f2783381a48c13c2f75c07e5029903cb. 2024-11-15T22:14:33,886 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,,1731708873681.f2783381a48c13c2f75c07e5029903cb. after waiting 0 ms 2024-11-15T22:14:33,886 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,,1731708873681.f2783381a48c13c2f75c07e5029903cb. 2024-11-15T22:14:33,886 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,,1731708873681.f2783381a48c13c2f75c07e5029903cb. 2024-11-15T22:14:33,886 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for f2783381a48c13c2f75c07e5029903cb: Waiting for close lock at 1731708873886Disabling compacts and flushes for region at 1731708873886Disabling writes for close at 1731708873886Writing region close event to WAL at 1731708873886Closed at 1731708873886 2024-11-15T22:14:33,887 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 51bf57bcf1afa9f625176363284d17f7, NAME => 'test,ddd,1731708873681.51bf57bcf1afa9f625176363284d17f7.', STARTKEY => 'ddd', ENDKEY => 'eee'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa 2024-11-15T22:14:33,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T22:14:33,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741840_1016 (size=45) 2024-11-15T22:14:33,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33631 is added to blk_1073741840_1016 (size=45) 2024-11-15T22:14:33,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39269 is added to blk_1073741840_1016 (size=45) 2024-11-15T22:14:33,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39269 is added to blk_1073741839_1015 (size=45) 2024-11-15T22:14:33,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741839_1015 (size=45) 2024-11-15T22:14:33,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33631 is added to blk_1073741839_1015 (size=45) 2024-11-15T22:14:33,903 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ccc,1731708873681.2424ff4b6d907aafacef6f661d88ef95.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:33,903 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 2424ff4b6d907aafacef6f661d88ef95, disabling compactions & flushes 2024-11-15T22:14:33,903 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ccc,1731708873681.2424ff4b6d907aafacef6f661d88ef95. 2024-11-15T22:14:33,903 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1731708873681.2424ff4b6d907aafacef6f661d88ef95. 2024-11-15T22:14:33,903 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1731708873681.2424ff4b6d907aafacef6f661d88ef95. after waiting 0 ms 2024-11-15T22:14:33,903 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1731708873681.2424ff4b6d907aafacef6f661d88ef95. 2024-11-15T22:14:33,903 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ccc,1731708873681.2424ff4b6d907aafacef6f661d88ef95. 2024-11-15T22:14:33,903 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 2424ff4b6d907aafacef6f661d88ef95: Waiting for close lock at 1731708873903Disabling compacts and flushes for region at 1731708873903Disabling writes for close at 1731708873903Writing region close event to WAL at 1731708873903Closed at 1731708873903 2024-11-15T22:14:33,904 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 93deaad3e3229244196e8daa9478371a, NAME => 'test,eee,1731708873681.93deaad3e3229244196e8daa9478371a.', STARTKEY => 'eee', ENDKEY => 'fff'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa 2024-11-15T22:14:33,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741841_1017 (size=45) 2024-11-15T22:14:33,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39269 is added to blk_1073741841_1017 (size=45) 2024-11-15T22:14:33,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33631 is added to blk_1073741841_1017 (size=45) 2024-11-15T22:14:33,917 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ddd,1731708873681.51bf57bcf1afa9f625176363284d17f7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:33,918 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 51bf57bcf1afa9f625176363284d17f7, disabling compactions & flushes 2024-11-15T22:14:33,918 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ddd,1731708873681.51bf57bcf1afa9f625176363284d17f7. 2024-11-15T22:14:33,918 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1731708873681.51bf57bcf1afa9f625176363284d17f7. 2024-11-15T22:14:33,918 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1731708873681.51bf57bcf1afa9f625176363284d17f7. after waiting 0 ms 2024-11-15T22:14:33,918 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1731708873681.51bf57bcf1afa9f625176363284d17f7. 2024-11-15T22:14:33,918 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ddd,1731708873681.51bf57bcf1afa9f625176363284d17f7. 2024-11-15T22:14:33,918 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 51bf57bcf1afa9f625176363284d17f7: Waiting for close lock at 1731708873918Disabling compacts and flushes for region at 1731708873918Disabling writes for close at 1731708873918Writing region close event to WAL at 1731708873918Closed at 1731708873918 2024-11-15T22:14:33,919 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ecaeddccb9b2987f43bb050f0a105976, NAME => 'test,fff,1731708873681.ecaeddccb9b2987f43bb050f0a105976.', STARTKEY => 'fff', ENDKEY => 'ggg'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa 2024-11-15T22:14:33,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741842_1018 (size=45) 2024-11-15T22:14:33,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39269 is added to blk_1073741842_1018 (size=45) 2024-11-15T22:14:33,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33631 is added to blk_1073741842_1018 (size=45) 2024-11-15T22:14:33,928 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,eee,1731708873681.93deaad3e3229244196e8daa9478371a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:33,928 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 93deaad3e3229244196e8daa9478371a, disabling compactions & flushes 2024-11-15T22:14:33,928 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,eee,1731708873681.93deaad3e3229244196e8daa9478371a. 2024-11-15T22:14:33,928 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1731708873681.93deaad3e3229244196e8daa9478371a. 2024-11-15T22:14:33,928 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,eee,1731708873681.93deaad3e3229244196e8daa9478371a. after waiting 0 ms 2024-11-15T22:14:33,928 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,eee,1731708873681.93deaad3e3229244196e8daa9478371a. 2024-11-15T22:14:33,928 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,eee,1731708873681.93deaad3e3229244196e8daa9478371a. 2024-11-15T22:14:33,928 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 93deaad3e3229244196e8daa9478371a: Waiting for close lock at 1731708873928Disabling compacts and flushes for region at 1731708873928Disabling writes for close at 1731708873928Writing region close event to WAL at 1731708873928Closed at 1731708873928 2024-11-15T22:14:33,929 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 1959cb95bb21ecdc677f7f5290173984, NAME => 'test,ggg,1731708873681.1959cb95bb21ecdc677f7f5290173984.', STARTKEY => 'ggg', ENDKEY => 'hhh'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa 2024-11-15T22:14:33,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741843_1019 (size=45) 2024-11-15T22:14:33,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33631 is added to blk_1073741843_1019 (size=45) 2024-11-15T22:14:33,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39269 is added to blk_1073741843_1019 (size=45) 2024-11-15T22:14:33,943 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,fff,1731708873681.ecaeddccb9b2987f43bb050f0a105976.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:33,943 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing ecaeddccb9b2987f43bb050f0a105976, disabling compactions & flushes 2024-11-15T22:14:33,943 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,fff,1731708873681.ecaeddccb9b2987f43bb050f0a105976. 2024-11-15T22:14:33,943 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1731708873681.ecaeddccb9b2987f43bb050f0a105976. 2024-11-15T22:14:33,943 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,fff,1731708873681.ecaeddccb9b2987f43bb050f0a105976. after waiting 0 ms 2024-11-15T22:14:33,944 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,fff,1731708873681.ecaeddccb9b2987f43bb050f0a105976. 2024-11-15T22:14:33,944 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,fff,1731708873681.ecaeddccb9b2987f43bb050f0a105976. 2024-11-15T22:14:33,944 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for ecaeddccb9b2987f43bb050f0a105976: Waiting for close lock at 1731708873943Disabling compacts and flushes for region at 1731708873943Disabling writes for close at 1731708873944 (+1 ms)Writing region close event to WAL at 1731708873944Closed at 1731708873944 2024-11-15T22:14:33,945 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 13c3457b9645c9a8181ed9b4551962ad, NAME => 'test,hhh,1731708873681.13c3457b9645c9a8181ed9b4551962ad.', STARTKEY => 'hhh', ENDKEY => 'iii'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa 2024-11-15T22:14:33,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39269 is added to blk_1073741844_1020 (size=45) 2024-11-15T22:14:33,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741844_1020 (size=45) 2024-11-15T22:14:33,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33631 is added to blk_1073741844_1020 (size=45) 2024-11-15T22:14:33,958 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ggg,1731708873681.1959cb95bb21ecdc677f7f5290173984.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:33,958 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 1959cb95bb21ecdc677f7f5290173984, disabling compactions & flushes 2024-11-15T22:14:33,959 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ggg,1731708873681.1959cb95bb21ecdc677f7f5290173984. 2024-11-15T22:14:33,959 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1731708873681.1959cb95bb21ecdc677f7f5290173984. 2024-11-15T22:14:33,959 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1731708873681.1959cb95bb21ecdc677f7f5290173984. after waiting 0 ms 2024-11-15T22:14:33,959 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1731708873681.1959cb95bb21ecdc677f7f5290173984. 2024-11-15T22:14:33,959 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ggg,1731708873681.1959cb95bb21ecdc677f7f5290173984. 2024-11-15T22:14:33,959 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 1959cb95bb21ecdc677f7f5290173984: Waiting for close lock at 1731708873958Disabling compacts and flushes for region at 1731708873958Disabling writes for close at 1731708873959 (+1 ms)Writing region close event to WAL at 1731708873959Closed at 1731708873959 2024-11-15T22:14:33,960 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 1732b30f3d981289760d152f788aff6f, NAME => 'test,iii,1731708873681.1732b30f3d981289760d152f788aff6f.', STARTKEY => 'iii', ENDKEY => 'jjj'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa 2024-11-15T22:14:33,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39269 is added to blk_1073741845_1021 (size=45) 2024-11-15T22:14:33,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741845_1021 (size=45) 2024-11-15T22:14:33,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33631 is added to blk_1073741845_1021 (size=45) 2024-11-15T22:14:33,964 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,hhh,1731708873681.13c3457b9645c9a8181ed9b4551962ad.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:33,964 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 13c3457b9645c9a8181ed9b4551962ad, disabling compactions & flushes 2024-11-15T22:14:33,964 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,hhh,1731708873681.13c3457b9645c9a8181ed9b4551962ad. 2024-11-15T22:14:33,964 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1731708873681.13c3457b9645c9a8181ed9b4551962ad. 2024-11-15T22:14:33,965 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1731708873681.13c3457b9645c9a8181ed9b4551962ad. after waiting 0 ms 2024-11-15T22:14:33,965 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1731708873681.13c3457b9645c9a8181ed9b4551962ad. 2024-11-15T22:14:33,965 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,hhh,1731708873681.13c3457b9645c9a8181ed9b4551962ad. 2024-11-15T22:14:33,965 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 13c3457b9645c9a8181ed9b4551962ad: Waiting for close lock at 1731708873964Disabling compacts and flushes for region at 1731708873964Disabling writes for close at 1731708873965 (+1 ms)Writing region close event to WAL at 1731708873965Closed at 1731708873965 2024-11-15T22:14:33,965 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 36b8b3bed3adb94f5901f1f3809b64fa, NAME => 'test,jjj,1731708873681.36b8b3bed3adb94f5901f1f3809b64fa.', STARTKEY => 'jjj', ENDKEY => 'kkk'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa 2024-11-15T22:14:33,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741846_1022 (size=45) 2024-11-15T22:14:33,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39269 is added to blk_1073741846_1022 (size=45) 2024-11-15T22:14:33,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33631 is added to blk_1073741846_1022 (size=45) 2024-11-15T22:14:33,985 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,iii,1731708873681.1732b30f3d981289760d152f788aff6f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:33,985 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 1732b30f3d981289760d152f788aff6f, disabling compactions & flushes 2024-11-15T22:14:33,985 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,iii,1731708873681.1732b30f3d981289760d152f788aff6f. 2024-11-15T22:14:33,985 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1731708873681.1732b30f3d981289760d152f788aff6f. 2024-11-15T22:14:33,985 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,iii,1731708873681.1732b30f3d981289760d152f788aff6f. after waiting 0 ms 2024-11-15T22:14:33,986 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,iii,1731708873681.1732b30f3d981289760d152f788aff6f. 2024-11-15T22:14:33,986 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,iii,1731708873681.1732b30f3d981289760d152f788aff6f. 2024-11-15T22:14:33,986 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 1732b30f3d981289760d152f788aff6f: Waiting for close lock at 1731708873985Disabling compacts and flushes for region at 1731708873985Disabling writes for close at 1731708873985Writing region close event to WAL at 1731708873986 (+1 ms)Closed at 1731708873986 2024-11-15T22:14:33,986 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 2a159d388ea2138ee8e2bed061a785ce, NAME => 'test,kkk,1731708873681.2a159d388ea2138ee8e2bed061a785ce.', STARTKEY => 'kkk', ENDKEY => 'lll'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa 2024-11-15T22:14:34,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39269 is added to blk_1073741847_1023 (size=45) 2024-11-15T22:14:34,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33631 is added to blk_1073741847_1023 (size=45) 2024-11-15T22:14:34,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741847_1023 (size=45) 2024-11-15T22:14:34,003 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,jjj,1731708873681.36b8b3bed3adb94f5901f1f3809b64fa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:34,004 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 36b8b3bed3adb94f5901f1f3809b64fa, disabling compactions & flushes 2024-11-15T22:14:34,004 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,jjj,1731708873681.36b8b3bed3adb94f5901f1f3809b64fa. 2024-11-15T22:14:34,004 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1731708873681.36b8b3bed3adb94f5901f1f3809b64fa. 2024-11-15T22:14:34,004 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1731708873681.36b8b3bed3adb94f5901f1f3809b64fa. after waiting 0 ms 2024-11-15T22:14:34,004 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1731708873681.36b8b3bed3adb94f5901f1f3809b64fa. 2024-11-15T22:14:34,004 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,jjj,1731708873681.36b8b3bed3adb94f5901f1f3809b64fa. 2024-11-15T22:14:34,004 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 36b8b3bed3adb94f5901f1f3809b64fa: Waiting for close lock at 1731708874004Disabling compacts and flushes for region at 1731708874004Disabling writes for close at 1731708874004Writing region close event to WAL at 1731708874004Closed at 1731708874004 2024-11-15T22:14:34,005 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => de9977b187fcaabda585b015a568e905, NAME => 'test,lll,1731708873681.de9977b187fcaabda585b015a568e905.', STARTKEY => 'lll', ENDKEY => 'mmm'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa 2024-11-15T22:14:34,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33631 is added to blk_1073741848_1024 (size=45) 2024-11-15T22:14:34,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39269 is added to blk_1073741848_1024 (size=45) 2024-11-15T22:14:34,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741848_1024 (size=45) 2024-11-15T22:14:34,024 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,kkk,1731708873681.2a159d388ea2138ee8e2bed061a785ce.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:34,024 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 2a159d388ea2138ee8e2bed061a785ce, disabling compactions & flushes 2024-11-15T22:14:34,024 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,kkk,1731708873681.2a159d388ea2138ee8e2bed061a785ce. 2024-11-15T22:14:34,024 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1731708873681.2a159d388ea2138ee8e2bed061a785ce. 2024-11-15T22:14:34,024 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1731708873681.2a159d388ea2138ee8e2bed061a785ce. after waiting 0 ms 2024-11-15T22:14:34,024 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1731708873681.2a159d388ea2138ee8e2bed061a785ce. 2024-11-15T22:14:34,024 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,kkk,1731708873681.2a159d388ea2138ee8e2bed061a785ce. 2024-11-15T22:14:34,024 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 2a159d388ea2138ee8e2bed061a785ce: Waiting for close lock at 1731708874024Disabling compacts and flushes for region at 1731708874024Disabling writes for close at 1731708874024Writing region close event to WAL at 1731708874024Closed at 1731708874024 2024-11-15T22:14:34,025 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => f64d8a749084ba25ea1b77db9a0d9f17, NAME => 'test,mmm,1731708873681.f64d8a749084ba25ea1b77db9a0d9f17.', STARTKEY => 'mmm', ENDKEY => 'nnn'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa 2024-11-15T22:14:34,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39269 is added to blk_1073741849_1025 (size=45) 2024-11-15T22:14:34,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33631 is added to blk_1073741849_1025 (size=45) 2024-11-15T22:14:34,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741849_1025 (size=45) 2024-11-15T22:14:34,039 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,lll,1731708873681.de9977b187fcaabda585b015a568e905.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:34,039 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing de9977b187fcaabda585b015a568e905, disabling compactions & flushes 2024-11-15T22:14:34,039 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,lll,1731708873681.de9977b187fcaabda585b015a568e905. 2024-11-15T22:14:34,040 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1731708873681.de9977b187fcaabda585b015a568e905. 2024-11-15T22:14:34,040 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,lll,1731708873681.de9977b187fcaabda585b015a568e905. after waiting 0 ms 2024-11-15T22:14:34,040 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,lll,1731708873681.de9977b187fcaabda585b015a568e905. 2024-11-15T22:14:34,040 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,lll,1731708873681.de9977b187fcaabda585b015a568e905. 2024-11-15T22:14:34,040 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for de9977b187fcaabda585b015a568e905: Waiting for close lock at 1731708874039Disabling compacts and flushes for region at 1731708874039Disabling writes for close at 1731708874040 (+1 ms)Writing region close event to WAL at 1731708874040Closed at 1731708874040 2024-11-15T22:14:34,041 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c56d152c07a95a189ac65341ff4db1fa, NAME => 'test,nnn,1731708873681.c56d152c07a95a189ac65341ff4db1fa.', STARTKEY => 'nnn', ENDKEY => 'ooo'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa 2024-11-15T22:14:34,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39269 is added to blk_1073741850_1026 (size=45) 2024-11-15T22:14:34,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741850_1026 (size=45) 2024-11-15T22:14:34,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33631 is added to blk_1073741850_1026 (size=45) 2024-11-15T22:14:34,054 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,mmm,1731708873681.f64d8a749084ba25ea1b77db9a0d9f17.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:34,054 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing f64d8a749084ba25ea1b77db9a0d9f17, disabling compactions & flushes 2024-11-15T22:14:34,054 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,mmm,1731708873681.f64d8a749084ba25ea1b77db9a0d9f17. 2024-11-15T22:14:34,054 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1731708873681.f64d8a749084ba25ea1b77db9a0d9f17. 2024-11-15T22:14:34,054 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1731708873681.f64d8a749084ba25ea1b77db9a0d9f17. after waiting 0 ms 2024-11-15T22:14:34,054 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1731708873681.f64d8a749084ba25ea1b77db9a0d9f17. 2024-11-15T22:14:34,054 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,mmm,1731708873681.f64d8a749084ba25ea1b77db9a0d9f17. 2024-11-15T22:14:34,054 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for f64d8a749084ba25ea1b77db9a0d9f17: Waiting for close lock at 1731708874054Disabling compacts and flushes for region at 1731708874054Disabling writes for close at 1731708874054Writing region close event to WAL at 1731708874054Closed at 1731708874054 2024-11-15T22:14:34,055 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => a602f0e8bd42c685584b8b985174ddea, NAME => 'test,ooo,1731708873681.a602f0e8bd42c685584b8b985174ddea.', STARTKEY => 'ooo', ENDKEY => 'ppp'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa 2024-11-15T22:14:34,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741851_1027 (size=45) 2024-11-15T22:14:34,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33631 is added to blk_1073741851_1027 (size=45) 2024-11-15T22:14:34,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39269 is added to blk_1073741851_1027 (size=45) 2024-11-15T22:14:34,075 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,nnn,1731708873681.c56d152c07a95a189ac65341ff4db1fa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:34,075 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing c56d152c07a95a189ac65341ff4db1fa, disabling compactions & flushes 2024-11-15T22:14:34,076 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,nnn,1731708873681.c56d152c07a95a189ac65341ff4db1fa. 2024-11-15T22:14:34,076 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1731708873681.c56d152c07a95a189ac65341ff4db1fa. 2024-11-15T22:14:34,076 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1731708873681.c56d152c07a95a189ac65341ff4db1fa. after waiting 0 ms 2024-11-15T22:14:34,076 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1731708873681.c56d152c07a95a189ac65341ff4db1fa. 2024-11-15T22:14:34,076 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,nnn,1731708873681.c56d152c07a95a189ac65341ff4db1fa. 2024-11-15T22:14:34,076 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for c56d152c07a95a189ac65341ff4db1fa: Waiting for close lock at 1731708874075Disabling compacts and flushes for region at 1731708874075Disabling writes for close at 1731708874076 (+1 ms)Writing region close event to WAL at 1731708874076Closed at 1731708874076 2024-11-15T22:14:34,076 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 83048e0be415b190a19f64bc2e49d3fe, NAME => 'test,ppp,1731708873681.83048e0be415b190a19f64bc2e49d3fe.', STARTKEY => 'ppp', ENDKEY => 'qqq'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa 2024-11-15T22:14:34,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39269 is added to blk_1073741853_1029 (size=45) 2024-11-15T22:14:34,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33631 is added to blk_1073741853_1029 (size=45) 2024-11-15T22:14:34,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741853_1029 (size=45) 2024-11-15T22:14:34,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T22:14:34,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741852_1028 (size=45) 2024-11-15T22:14:34,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33631 is added to blk_1073741852_1028 (size=45) 2024-11-15T22:14:34,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39269 is added to blk_1073741852_1028 (size=45) 2024-11-15T22:14:34,104 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ooo,1731708873681.a602f0e8bd42c685584b8b985174ddea.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:34,104 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing a602f0e8bd42c685584b8b985174ddea, disabling compactions & flushes 2024-11-15T22:14:34,104 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ooo,1731708873681.a602f0e8bd42c685584b8b985174ddea. 2024-11-15T22:14:34,104 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1731708873681.a602f0e8bd42c685584b8b985174ddea. 2024-11-15T22:14:34,105 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1731708873681.a602f0e8bd42c685584b8b985174ddea. after waiting 0 ms 2024-11-15T22:14:34,105 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1731708873681.a602f0e8bd42c685584b8b985174ddea. 2024-11-15T22:14:34,105 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ooo,1731708873681.a602f0e8bd42c685584b8b985174ddea. 2024-11-15T22:14:34,105 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for a602f0e8bd42c685584b8b985174ddea: Waiting for close lock at 1731708874104Disabling compacts and flushes for region at 1731708874104Disabling writes for close at 1731708874105 (+1 ms)Writing region close event to WAL at 1731708874105Closed at 1731708874105 2024-11-15T22:14:34,105 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 06b9df8f55d791077e9f7ec55ef6a971, NAME => 'test,qqq,1731708873681.06b9df8f55d791077e9f7ec55ef6a971.', STARTKEY => 'qqq', ENDKEY => 'rrr'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa 2024-11-15T22:14:34,108 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ppp,1731708873681.83048e0be415b190a19f64bc2e49d3fe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:34,108 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 83048e0be415b190a19f64bc2e49d3fe, disabling compactions & flushes 2024-11-15T22:14:34,109 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ppp,1731708873681.83048e0be415b190a19f64bc2e49d3fe. 2024-11-15T22:14:34,109 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1731708873681.83048e0be415b190a19f64bc2e49d3fe. 2024-11-15T22:14:34,109 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1731708873681.83048e0be415b190a19f64bc2e49d3fe. after waiting 0 ms 2024-11-15T22:14:34,109 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1731708873681.83048e0be415b190a19f64bc2e49d3fe. 2024-11-15T22:14:34,109 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ppp,1731708873681.83048e0be415b190a19f64bc2e49d3fe. 2024-11-15T22:14:34,109 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 83048e0be415b190a19f64bc2e49d3fe: Waiting for close lock at 1731708874108Disabling compacts and flushes for region at 1731708874108Disabling writes for close at 1731708874109 (+1 ms)Writing region close event to WAL at 1731708874109Closed at 1731708874109 2024-11-15T22:14:34,110 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7502fc847b4b97e1ace9a22ec1c068e0, NAME => 'test,rrr,1731708873681.7502fc847b4b97e1ace9a22ec1c068e0.', STARTKEY => 'rrr', ENDKEY => 'sss'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa 2024-11-15T22:14:34,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741854_1030 (size=45) 2024-11-15T22:14:34,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33631 is added to blk_1073741854_1030 (size=45) 2024-11-15T22:14:34,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39269 is added to blk_1073741854_1030 (size=45) 2024-11-15T22:14:34,126 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,qqq,1731708873681.06b9df8f55d791077e9f7ec55ef6a971.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:34,126 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 06b9df8f55d791077e9f7ec55ef6a971, disabling compactions & flushes 2024-11-15T22:14:34,126 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,qqq,1731708873681.06b9df8f55d791077e9f7ec55ef6a971. 2024-11-15T22:14:34,126 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1731708873681.06b9df8f55d791077e9f7ec55ef6a971. 2024-11-15T22:14:34,126 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1731708873681.06b9df8f55d791077e9f7ec55ef6a971. after waiting 0 ms 2024-11-15T22:14:34,126 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1731708873681.06b9df8f55d791077e9f7ec55ef6a971. 2024-11-15T22:14:34,126 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,qqq,1731708873681.06b9df8f55d791077e9f7ec55ef6a971. 2024-11-15T22:14:34,126 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 06b9df8f55d791077e9f7ec55ef6a971: Waiting for close lock at 1731708874126Disabling compacts and flushes for region at 1731708874126Disabling writes for close at 1731708874126Writing region close event to WAL at 1731708874126Closed at 1731708874126 2024-11-15T22:14:34,127 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 819421a6406c0ebb5635c399b2c0303d, NAME => 'test,sss,1731708873681.819421a6406c0ebb5635c399b2c0303d.', STARTKEY => 'sss', ENDKEY => 'ttt'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa 2024-11-15T22:14:34,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741855_1031 (size=45) 2024-11-15T22:14:34,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33631 is added to blk_1073741855_1031 (size=45) 2024-11-15T22:14:34,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39269 is added to blk_1073741855_1031 (size=45) 2024-11-15T22:14:34,137 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,rrr,1731708873681.7502fc847b4b97e1ace9a22ec1c068e0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:34,137 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 7502fc847b4b97e1ace9a22ec1c068e0, disabling compactions & flushes 2024-11-15T22:14:34,137 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,rrr,1731708873681.7502fc847b4b97e1ace9a22ec1c068e0. 2024-11-15T22:14:34,137 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1731708873681.7502fc847b4b97e1ace9a22ec1c068e0. 2024-11-15T22:14:34,137 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1731708873681.7502fc847b4b97e1ace9a22ec1c068e0. after waiting 0 ms 2024-11-15T22:14:34,137 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1731708873681.7502fc847b4b97e1ace9a22ec1c068e0. 2024-11-15T22:14:34,137 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,rrr,1731708873681.7502fc847b4b97e1ace9a22ec1c068e0. 2024-11-15T22:14:34,138 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7502fc847b4b97e1ace9a22ec1c068e0: Waiting for close lock at 1731708874137Disabling compacts and flushes for region at 1731708874137Disabling writes for close at 1731708874137Writing region close event to WAL at 1731708874137Closed at 1731708874137 2024-11-15T22:14:34,138 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4381d14830a288bee3aeb42195dd8768, NAME => 'test,ttt,1731708873681.4381d14830a288bee3aeb42195dd8768.', STARTKEY => 'ttt', ENDKEY => 'uuu'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa 2024-11-15T22:14:34,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741857_1033 (size=45) 2024-11-15T22:14:34,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39269 is added to blk_1073741857_1033 (size=45) 2024-11-15T22:14:34,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33631 is added to blk_1073741857_1033 (size=45) 2024-11-15T22:14:34,157 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ttt,1731708873681.4381d14830a288bee3aeb42195dd8768.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:34,158 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 4381d14830a288bee3aeb42195dd8768, disabling compactions & flushes 2024-11-15T22:14:34,158 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ttt,1731708873681.4381d14830a288bee3aeb42195dd8768. 2024-11-15T22:14:34,158 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1731708873681.4381d14830a288bee3aeb42195dd8768. 2024-11-15T22:14:34,158 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1731708873681.4381d14830a288bee3aeb42195dd8768. after waiting 0 ms 2024-11-15T22:14:34,158 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1731708873681.4381d14830a288bee3aeb42195dd8768. 2024-11-15T22:14:34,158 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ttt,1731708873681.4381d14830a288bee3aeb42195dd8768. 2024-11-15T22:14:34,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741856_1032 (size=45) 2024-11-15T22:14:34,158 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4381d14830a288bee3aeb42195dd8768: Waiting for close lock at 1731708874158Disabling compacts and flushes for region at 1731708874158Disabling writes for close at 1731708874158Writing region close event to WAL at 1731708874158Closed at 1731708874158 2024-11-15T22:14:34,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33631 is added to blk_1073741856_1032 (size=45) 2024-11-15T22:14:34,159 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e87a82b283b7251342a664ba3876c98e, NAME => 'test,uuu,1731708873681.e87a82b283b7251342a664ba3876c98e.', STARTKEY => 'uuu', ENDKEY => 'vvv'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa 2024-11-15T22:14:34,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39269 is added to blk_1073741856_1032 (size=45) 2024-11-15T22:14:34,162 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,sss,1731708873681.819421a6406c0ebb5635c399b2c0303d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:34,162 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 819421a6406c0ebb5635c399b2c0303d, disabling compactions & flushes 2024-11-15T22:14:34,162 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,sss,1731708873681.819421a6406c0ebb5635c399b2c0303d. 2024-11-15T22:14:34,162 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1731708873681.819421a6406c0ebb5635c399b2c0303d. 2024-11-15T22:14:34,162 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,sss,1731708873681.819421a6406c0ebb5635c399b2c0303d. after waiting 0 ms 2024-11-15T22:14:34,162 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,sss,1731708873681.819421a6406c0ebb5635c399b2c0303d. 2024-11-15T22:14:34,162 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,sss,1731708873681.819421a6406c0ebb5635c399b2c0303d. 2024-11-15T22:14:34,163 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 819421a6406c0ebb5635c399b2c0303d: Waiting for close lock at 1731708874162Disabling compacts and flushes for region at 1731708874162Disabling writes for close at 1731708874162Writing region close event to WAL at 1731708874162Closed at 1731708874162 2024-11-15T22:14:34,163 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => f9e6155d6c82d9d9a5c9449be6d7ce5f, NAME => 'test,vvv,1731708873681.f9e6155d6c82d9d9a5c9449be6d7ce5f.', STARTKEY => 'vvv', ENDKEY => 'www'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa 2024-11-15T22:14:34,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741859_1035 (size=45) 2024-11-15T22:14:34,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33631 is added to blk_1073741859_1035 (size=45) 2024-11-15T22:14:34,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39269 is added to blk_1073741859_1035 (size=45) 2024-11-15T22:14:34,193 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,vvv,1731708873681.f9e6155d6c82d9d9a5c9449be6d7ce5f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:34,193 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing f9e6155d6c82d9d9a5c9449be6d7ce5f, disabling compactions & flushes 2024-11-15T22:14:34,193 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,vvv,1731708873681.f9e6155d6c82d9d9a5c9449be6d7ce5f. 2024-11-15T22:14:34,193 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1731708873681.f9e6155d6c82d9d9a5c9449be6d7ce5f. 2024-11-15T22:14:34,193 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1731708873681.f9e6155d6c82d9d9a5c9449be6d7ce5f. after waiting 0 ms 2024-11-15T22:14:34,194 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1731708873681.f9e6155d6c82d9d9a5c9449be6d7ce5f. 2024-11-15T22:14:34,194 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,vvv,1731708873681.f9e6155d6c82d9d9a5c9449be6d7ce5f. 2024-11-15T22:14:34,194 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for f9e6155d6c82d9d9a5c9449be6d7ce5f: Waiting for close lock at 1731708874193Disabling compacts and flushes for region at 1731708874193Disabling writes for close at 1731708874194 (+1 ms)Writing region close event to WAL at 1731708874194Closed at 1731708874194 2024-11-15T22:14:34,194 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 2e79a8ce6f9715045d056d3dfdef5cec, NAME => 'test,www,1731708873681.2e79a8ce6f9715045d056d3dfdef5cec.', STARTKEY => 'www', ENDKEY => 'xxx'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa 2024-11-15T22:14:34,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39269 is added to blk_1073741858_1034 (size=45) 2024-11-15T22:14:34,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741858_1034 (size=45) 2024-11-15T22:14:34,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39269 is added to blk_1073741860_1036 (size=45) 2024-11-15T22:14:34,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33631 is added to blk_1073741858_1034 (size=45) 2024-11-15T22:14:34,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33631 is added to blk_1073741860_1036 (size=45) 2024-11-15T22:14:34,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741860_1036 (size=45) 2024-11-15T22:14:34,209 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,uuu,1731708873681.e87a82b283b7251342a664ba3876c98e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:34,209 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,www,1731708873681.2e79a8ce6f9715045d056d3dfdef5cec.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:34,209 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing e87a82b283b7251342a664ba3876c98e, disabling compactions & flushes 2024-11-15T22:14:34,209 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,uuu,1731708873681.e87a82b283b7251342a664ba3876c98e. 2024-11-15T22:14:34,210 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 2e79a8ce6f9715045d056d3dfdef5cec, disabling compactions & flushes 2024-11-15T22:14:34,210 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1731708873681.e87a82b283b7251342a664ba3876c98e. 2024-11-15T22:14:34,210 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,www,1731708873681.2e79a8ce6f9715045d056d3dfdef5cec. 2024-11-15T22:14:34,210 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1731708873681.e87a82b283b7251342a664ba3876c98e. after waiting 0 ms 2024-11-15T22:14:34,210 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1731708873681.2e79a8ce6f9715045d056d3dfdef5cec. 2024-11-15T22:14:34,210 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1731708873681.e87a82b283b7251342a664ba3876c98e. 2024-11-15T22:14:34,210 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,www,1731708873681.2e79a8ce6f9715045d056d3dfdef5cec. after waiting 0 ms 2024-11-15T22:14:34,210 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,www,1731708873681.2e79a8ce6f9715045d056d3dfdef5cec. 2024-11-15T22:14:34,210 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,uuu,1731708873681.e87a82b283b7251342a664ba3876c98e. 2024-11-15T22:14:34,210 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,www,1731708873681.2e79a8ce6f9715045d056d3dfdef5cec. 2024-11-15T22:14:34,210 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for e87a82b283b7251342a664ba3876c98e: Waiting for close lock at 1731708874209Disabling compacts and flushes for region at 1731708874209Disabling writes for close at 1731708874210 (+1 ms)Writing region close event to WAL at 1731708874210Closed at 1731708874210 2024-11-15T22:14:34,210 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 2e79a8ce6f9715045d056d3dfdef5cec: Waiting for close lock at 1731708874209Disabling compacts and flushes for region at 1731708874209Disabling writes for close at 1731708874210 (+1 ms)Writing region close event to WAL at 1731708874210Closed at 1731708874210 2024-11-15T22:14:34,210 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a7fb991e3993a690fe7a0a1b064d5f5d, NAME => 'test,xxx,1731708873681.a7fb991e3993a690fe7a0a1b064d5f5d.', STARTKEY => 'xxx', ENDKEY => 'yyy'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa 2024-11-15T22:14:34,211 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => c1cecce90107a4c37b7d91651975993f, NAME => 'test,yyy,1731708873681.c1cecce90107a4c37b7d91651975993f.', STARTKEY => 'yyy', ENDKEY => 'zzz'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa 2024-11-15T22:14:34,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741861_1037 (size=45) 2024-11-15T22:14:34,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39269 is added to blk_1073741861_1037 (size=45) 2024-11-15T22:14:34,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33631 is added to blk_1073741861_1037 (size=45) 2024-11-15T22:14:34,245 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,yyy,1731708873681.c1cecce90107a4c37b7d91651975993f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:34,245 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing c1cecce90107a4c37b7d91651975993f, disabling compactions & flushes 2024-11-15T22:14:34,245 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,yyy,1731708873681.c1cecce90107a4c37b7d91651975993f. 2024-11-15T22:14:34,245 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1731708873681.c1cecce90107a4c37b7d91651975993f. 2024-11-15T22:14:34,245 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1731708873681.c1cecce90107a4c37b7d91651975993f. after waiting 0 ms 2024-11-15T22:14:34,245 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1731708873681.c1cecce90107a4c37b7d91651975993f. 2024-11-15T22:14:34,245 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,yyy,1731708873681.c1cecce90107a4c37b7d91651975993f. 2024-11-15T22:14:34,245 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for c1cecce90107a4c37b7d91651975993f: Waiting for close lock at 1731708874245Disabling compacts and flushes for region at 1731708874245Disabling writes for close at 1731708874245Writing region close event to WAL at 1731708874245Closed at 1731708874245 2024-11-15T22:14:34,246 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 6efdb15046141ab0cd574540ef2ac2c2, NAME => 'test,zzz,1731708873681.6efdb15046141ab0cd574540ef2ac2c2.', STARTKEY => 'zzz', ENDKEY => ''}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa 2024-11-15T22:14:34,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741862_1038 (size=45) 2024-11-15T22:14:34,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33631 is added to blk_1073741862_1038 (size=45) 2024-11-15T22:14:34,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39269 is added to blk_1073741862_1038 (size=45) 2024-11-15T22:14:34,274 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,xxx,1731708873681.a7fb991e3993a690fe7a0a1b064d5f5d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:34,274 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing a7fb991e3993a690fe7a0a1b064d5f5d, disabling compactions & flushes 2024-11-15T22:14:34,274 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,xxx,1731708873681.a7fb991e3993a690fe7a0a1b064d5f5d. 2024-11-15T22:14:34,274 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1731708873681.a7fb991e3993a690fe7a0a1b064d5f5d. 2024-11-15T22:14:34,274 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1731708873681.a7fb991e3993a690fe7a0a1b064d5f5d. after waiting 0 ms 2024-11-15T22:14:34,274 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1731708873681.a7fb991e3993a690fe7a0a1b064d5f5d. 2024-11-15T22:14:34,274 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,xxx,1731708873681.a7fb991e3993a690fe7a0a1b064d5f5d. 2024-11-15T22:14:34,274 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for a7fb991e3993a690fe7a0a1b064d5f5d: Waiting for close lock at 1731708874274Disabling compacts and flushes for region at 1731708874274Disabling writes for close at 1731708874274Writing region close event to WAL at 1731708874274Closed at 1731708874274 2024-11-15T22:14:34,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741863_1039 (size=42) 2024-11-15T22:14:34,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39269 is added to blk_1073741863_1039 (size=42) 2024-11-15T22:14:34,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33631 is added to blk_1073741863_1039 (size=42) 2024-11-15T22:14:34,298 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,bbb,1731708873681.897a7bb57a002d365489ce583eb7f84f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:34,298 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,zzz,1731708873681.6efdb15046141ab0cd574540ef2ac2c2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:34,298 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 897a7bb57a002d365489ce583eb7f84f, disabling compactions & flushes 2024-11-15T22:14:34,298 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,bbb,1731708873681.897a7bb57a002d365489ce583eb7f84f. 2024-11-15T22:14:34,298 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 6efdb15046141ab0cd574540ef2ac2c2, disabling compactions & flushes 2024-11-15T22:14:34,298 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1731708873681.897a7bb57a002d365489ce583eb7f84f. 2024-11-15T22:14:34,298 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1731708873681.897a7bb57a002d365489ce583eb7f84f. after waiting 0 ms 2024-11-15T22:14:34,298 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,zzz,1731708873681.6efdb15046141ab0cd574540ef2ac2c2. 2024-11-15T22:14:34,298 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1731708873681.897a7bb57a002d365489ce583eb7f84f. 2024-11-15T22:14:34,298 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1731708873681.6efdb15046141ab0cd574540ef2ac2c2. 2024-11-15T22:14:34,298 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,bbb,1731708873681.897a7bb57a002d365489ce583eb7f84f. 2024-11-15T22:14:34,298 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1731708873681.6efdb15046141ab0cd574540ef2ac2c2. after waiting 0 ms 2024-11-15T22:14:34,298 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 897a7bb57a002d365489ce583eb7f84f: Waiting for close lock at 1731708874298Disabling compacts and flushes for region at 1731708874298Disabling writes for close at 1731708874298Writing region close event to WAL at 1731708874298Closed at 1731708874298 2024-11-15T22:14:34,298 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1731708873681.6efdb15046141ab0cd574540ef2ac2c2. 2024-11-15T22:14:34,298 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,zzz,1731708873681.6efdb15046141ab0cd574540ef2ac2c2. 2024-11-15T22:14:34,298 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 6efdb15046141ab0cd574540ef2ac2c2: Waiting for close lock at 1731708874298Disabling compacts and flushes for region at 1731708874298Disabling writes for close at 1731708874298Writing region close event to WAL at 1731708874298Closed at 1731708874298 2024-11-15T22:14:34,313 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T22:14:34,319 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,,1731708873681.f2783381a48c13c2f75c07e5029903cb.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1731708874313"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731708874313"}]},"ts":"1731708874313"} 2024-11-15T22:14:34,319 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ccc,1731708873681.2424ff4b6d907aafacef6f661d88ef95.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731708874313"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731708874313"}]},"ts":"1731708874313"} 2024-11-15T22:14:34,320 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ddd,1731708873681.51bf57bcf1afa9f625176363284d17f7.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731708874313"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731708874313"}]},"ts":"1731708874313"} 2024-11-15T22:14:34,320 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,eee,1731708873681.93deaad3e3229244196e8daa9478371a.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731708874313"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731708874313"}]},"ts":"1731708874313"} 2024-11-15T22:14:34,320 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,fff,1731708873681.ecaeddccb9b2987f43bb050f0a105976.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731708874313"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731708874313"}]},"ts":"1731708874313"} 2024-11-15T22:14:34,320 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ggg,1731708873681.1959cb95bb21ecdc677f7f5290173984.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731708874313"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731708874313"}]},"ts":"1731708874313"} 2024-11-15T22:14:34,320 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,hhh,1731708873681.13c3457b9645c9a8181ed9b4551962ad.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731708874313"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731708874313"}]},"ts":"1731708874313"} 2024-11-15T22:14:34,321 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,iii,1731708873681.1732b30f3d981289760d152f788aff6f.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731708874313"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731708874313"}]},"ts":"1731708874313"} 2024-11-15T22:14:34,321 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,jjj,1731708873681.36b8b3bed3adb94f5901f1f3809b64fa.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731708874313"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731708874313"}]},"ts":"1731708874313"} 2024-11-15T22:14:34,321 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,kkk,1731708873681.2a159d388ea2138ee8e2bed061a785ce.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731708874313"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731708874313"}]},"ts":"1731708874313"} 2024-11-15T22:14:34,321 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,lll,1731708873681.de9977b187fcaabda585b015a568e905.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731708874313"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731708874313"}]},"ts":"1731708874313"} 2024-11-15T22:14:34,322 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,mmm,1731708873681.f64d8a749084ba25ea1b77db9a0d9f17.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731708874313"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731708874313"}]},"ts":"1731708874313"} 2024-11-15T22:14:34,322 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,nnn,1731708873681.c56d152c07a95a189ac65341ff4db1fa.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731708874313"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731708874313"}]},"ts":"1731708874313"} 2024-11-15T22:14:34,322 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ooo,1731708873681.a602f0e8bd42c685584b8b985174ddea.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731708874313"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731708874313"}]},"ts":"1731708874313"} 2024-11-15T22:14:34,322 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ppp,1731708873681.83048e0be415b190a19f64bc2e49d3fe.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731708874313"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731708874313"}]},"ts":"1731708874313"} 2024-11-15T22:14:34,323 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,qqq,1731708873681.06b9df8f55d791077e9f7ec55ef6a971.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731708874313"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731708874313"}]},"ts":"1731708874313"} 2024-11-15T22:14:34,323 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,rrr,1731708873681.7502fc847b4b97e1ace9a22ec1c068e0.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731708874313"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731708874313"}]},"ts":"1731708874313"} 2024-11-15T22:14:34,323 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ttt,1731708873681.4381d14830a288bee3aeb42195dd8768.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731708874313"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731708874313"}]},"ts":"1731708874313"} 2024-11-15T22:14:34,323 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,sss,1731708873681.819421a6406c0ebb5635c399b2c0303d.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731708874313"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731708874313"}]},"ts":"1731708874313"} 2024-11-15T22:14:34,323 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,vvv,1731708873681.f9e6155d6c82d9d9a5c9449be6d7ce5f.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731708874313"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731708874313"}]},"ts":"1731708874313"} 2024-11-15T22:14:34,323 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,uuu,1731708873681.e87a82b283b7251342a664ba3876c98e.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731708874313"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731708874313"}]},"ts":"1731708874313"} 2024-11-15T22:14:34,324 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,www,1731708873681.2e79a8ce6f9715045d056d3dfdef5cec.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731708874313"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731708874313"}]},"ts":"1731708874313"} 2024-11-15T22:14:34,324 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,yyy,1731708873681.c1cecce90107a4c37b7d91651975993f.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731708874313"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731708874313"}]},"ts":"1731708874313"} 2024-11-15T22:14:34,324 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,xxx,1731708873681.a7fb991e3993a690fe7a0a1b064d5f5d.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731708874313"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731708874313"}]},"ts":"1731708874313"} 2024-11-15T22:14:34,324 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,bbb,1731708873681.897a7bb57a002d365489ce583eb7f84f.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731708874313"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731708874313"}]},"ts":"1731708874313"} 2024-11-15T22:14:34,324 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,zzz,1731708873681.6efdb15046141ab0cd574540ef2ac2c2.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1731708874313"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731708874313"}]},"ts":"1731708874313"} 2024-11-15T22:14:34,383 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 26 regions to meta. 2024-11-15T22:14:34,386 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T22:14:34,388 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731708874386"}]},"ts":"1731708874386"} 2024-11-15T22:14:34,393 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLING in hbase:meta 2024-11-15T22:14:34,394 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {e611192d6313=0} racks are {/default-rack=0} 2024-11-15T22:14:34,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T22:14:34,417 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-15T22:14:34,417 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-15T22:14:34,417 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-15T22:14:34,417 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-15T22:14:34,417 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-15T22:14:34,417 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-15T22:14:34,417 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-15T22:14:34,417 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-15T22:14:34,417 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-15T22:14:34,417 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-15T22:14:34,421 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f2783381a48c13c2f75c07e5029903cb, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=897a7bb57a002d365489ce583eb7f84f, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2424ff4b6d907aafacef6f661d88ef95, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=51bf57bcf1afa9f625176363284d17f7, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=93deaad3e3229244196e8daa9478371a, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ecaeddccb9b2987f43bb050f0a105976, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1959cb95bb21ecdc677f7f5290173984, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1732b30f3d981289760d152f788aff6f, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=36b8b3bed3adb94f5901f1f3809b64fa, ASSIGN}, {pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN}, {pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=de9977b187fcaabda585b015a568e905, ASSIGN}, {pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f64d8a749084ba25ea1b77db9a0d9f17, ASSIGN}, {pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c56d152c07a95a189ac65341ff4db1fa, ASSIGN}, {pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a602f0e8bd42c685584b8b985174ddea, ASSIGN}, {pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=83048e0be415b190a19f64bc2e49d3fe, ASSIGN}, {pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=06b9df8f55d791077e9f7ec55ef6a971, ASSIGN}, {pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7502fc847b4b97e1ace9a22ec1c068e0, ASSIGN}, {pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN}, {pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4381d14830a288bee3aeb42195dd8768, ASSIGN}, {pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e87a82b283b7251342a664ba3876c98e, ASSIGN}, {pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f9e6155d6c82d9d9a5c9449be6d7ce5f, ASSIGN}, {pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2e79a8ce6f9715045d056d3dfdef5cec, ASSIGN}, {pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a7fb991e3993a690fe7a0a1b064d5f5d, ASSIGN}, {pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c1cecce90107a4c37b7d91651975993f, ASSIGN}, {pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6efdb15046141ab0cd574540ef2ac2c2, ASSIGN}] 2024-11-15T22:14:34,432 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6efdb15046141ab0cd574540ef2ac2c2, ASSIGN 2024-11-15T22:14:34,434 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=6efdb15046141ab0cd574540ef2ac2c2, ASSIGN; state=OFFLINE, location=e611192d6313,38729,1731708868111; forceNewPlan=false, retain=false 2024-11-15T22:14:34,436 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c1cecce90107a4c37b7d91651975993f, ASSIGN 2024-11-15T22:14:34,438 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c1cecce90107a4c37b7d91651975993f, ASSIGN; state=OFFLINE, location=e611192d6313,38729,1731708868111; forceNewPlan=false, retain=false 2024-11-15T22:14:34,440 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a7fb991e3993a690fe7a0a1b064d5f5d, ASSIGN 2024-11-15T22:14:34,443 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a7fb991e3993a690fe7a0a1b064d5f5d, ASSIGN; state=OFFLINE, location=e611192d6313,41231,1731708869041; forceNewPlan=false, retain=false 2024-11-15T22:14:34,446 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2e79a8ce6f9715045d056d3dfdef5cec, ASSIGN 2024-11-15T22:14:34,448 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=2e79a8ce6f9715045d056d3dfdef5cec, ASSIGN; state=OFFLINE, location=e611192d6313,43349,1731708869385; forceNewPlan=false, retain=false 2024-11-15T22:14:34,450 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f9e6155d6c82d9d9a5c9449be6d7ce5f, ASSIGN 2024-11-15T22:14:34,452 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f9e6155d6c82d9d9a5c9449be6d7ce5f, ASSIGN; state=OFFLINE, location=e611192d6313,38729,1731708868111; forceNewPlan=false, retain=false 2024-11-15T22:14:34,455 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e87a82b283b7251342a664ba3876c98e, ASSIGN 2024-11-15T22:14:34,457 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=e87a82b283b7251342a664ba3876c98e, ASSIGN; state=OFFLINE, location=e611192d6313,41231,1731708869041; forceNewPlan=false, retain=false 2024-11-15T22:14:34,459 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4381d14830a288bee3aeb42195dd8768, ASSIGN 2024-11-15T22:14:34,462 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4381d14830a288bee3aeb42195dd8768, ASSIGN; state=OFFLINE, location=e611192d6313,41231,1731708869041; forceNewPlan=false, retain=false 2024-11-15T22:14:34,464 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN 2024-11-15T22:14:34,466 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN; state=OFFLINE, location=e611192d6313,43349,1731708869385; forceNewPlan=false, retain=false 2024-11-15T22:14:34,470 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7502fc847b4b97e1ace9a22ec1c068e0, ASSIGN 2024-11-15T22:14:34,473 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=7502fc847b4b97e1ace9a22ec1c068e0, ASSIGN; state=OFFLINE, location=e611192d6313,38729,1731708868111; forceNewPlan=false, retain=false 2024-11-15T22:14:34,475 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=06b9df8f55d791077e9f7ec55ef6a971, ASSIGN 2024-11-15T22:14:34,478 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=06b9df8f55d791077e9f7ec55ef6a971, ASSIGN; state=OFFLINE, location=e611192d6313,43349,1731708869385; forceNewPlan=false, retain=false 2024-11-15T22:14:34,480 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=83048e0be415b190a19f64bc2e49d3fe, ASSIGN 2024-11-15T22:14:34,482 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=83048e0be415b190a19f64bc2e49d3fe, ASSIGN; state=OFFLINE, location=e611192d6313,41231,1731708869041; forceNewPlan=false, retain=false 2024-11-15T22:14:34,485 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a602f0e8bd42c685584b8b985174ddea, ASSIGN 2024-11-15T22:14:34,487 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a602f0e8bd42c685584b8b985174ddea, ASSIGN; state=OFFLINE, location=e611192d6313,38729,1731708868111; forceNewPlan=false, retain=false 2024-11-15T22:14:34,489 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c56d152c07a95a189ac65341ff4db1fa, ASSIGN 2024-11-15T22:14:34,490 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c56d152c07a95a189ac65341ff4db1fa, ASSIGN; state=OFFLINE, location=e611192d6313,43349,1731708869385; forceNewPlan=false, retain=false 2024-11-15T22:14:34,492 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f64d8a749084ba25ea1b77db9a0d9f17, ASSIGN 2024-11-15T22:14:34,494 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f64d8a749084ba25ea1b77db9a0d9f17, ASSIGN; state=OFFLINE, location=e611192d6313,41231,1731708869041; forceNewPlan=false, retain=false 2024-11-15T22:14:34,496 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=de9977b187fcaabda585b015a568e905, ASSIGN 2024-11-15T22:14:34,498 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=de9977b187fcaabda585b015a568e905, ASSIGN; state=OFFLINE, location=e611192d6313,38729,1731708868111; forceNewPlan=false, retain=false 2024-11-15T22:14:34,500 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN 2024-11-15T22:14:34,503 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN; state=OFFLINE, location=e611192d6313,43349,1731708869385; forceNewPlan=false, retain=false 2024-11-15T22:14:34,505 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=36b8b3bed3adb94f5901f1f3809b64fa, ASSIGN 2024-11-15T22:14:34,510 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=36b8b3bed3adb94f5901f1f3809b64fa, ASSIGN; state=OFFLINE, location=e611192d6313,41231,1731708869041; forceNewPlan=false, retain=false 2024-11-15T22:14:34,512 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1732b30f3d981289760d152f788aff6f, ASSIGN 2024-11-15T22:14:34,514 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=1732b30f3d981289760d152f788aff6f, ASSIGN; state=OFFLINE, location=e611192d6313,38729,1731708868111; forceNewPlan=false, retain=false 2024-11-15T22:14:34,517 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN 2024-11-15T22:14:34,520 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN; state=OFFLINE, location=e611192d6313,43349,1731708869385; forceNewPlan=false, retain=false 2024-11-15T22:14:34,522 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1959cb95bb21ecdc677f7f5290173984, ASSIGN 2024-11-15T22:14:34,525 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=1959cb95bb21ecdc677f7f5290173984, ASSIGN; state=OFFLINE, location=e611192d6313,41231,1731708869041; forceNewPlan=false, retain=false 2024-11-15T22:14:34,530 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ecaeddccb9b2987f43bb050f0a105976, ASSIGN 2024-11-15T22:14:34,532 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ecaeddccb9b2987f43bb050f0a105976, ASSIGN; state=OFFLINE, location=e611192d6313,38729,1731708868111; forceNewPlan=false, retain=false 2024-11-15T22:14:34,537 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=93deaad3e3229244196e8daa9478371a, ASSIGN 2024-11-15T22:14:34,540 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=93deaad3e3229244196e8daa9478371a, ASSIGN; state=OFFLINE, location=e611192d6313,43349,1731708869385; forceNewPlan=false, retain=false 2024-11-15T22:14:34,543 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=51bf57bcf1afa9f625176363284d17f7, ASSIGN 2024-11-15T22:14:34,553 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=51bf57bcf1afa9f625176363284d17f7, ASSIGN; state=OFFLINE, location=e611192d6313,41231,1731708869041; forceNewPlan=false, retain=false 2024-11-15T22:14:34,557 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2424ff4b6d907aafacef6f661d88ef95, ASSIGN 2024-11-15T22:14:34,561 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=2424ff4b6d907aafacef6f661d88ef95, ASSIGN; state=OFFLINE, location=e611192d6313,38729,1731708868111; forceNewPlan=false, retain=false 2024-11-15T22:14:34,566 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=897a7bb57a002d365489ce583eb7f84f, ASSIGN 2024-11-15T22:14:34,568 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=897a7bb57a002d365489ce583eb7f84f, ASSIGN; state=OFFLINE, location=e611192d6313,43349,1731708869385; forceNewPlan=false, retain=false 2024-11-15T22:14:34,571 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f2783381a48c13c2f75c07e5029903cb, ASSIGN 2024-11-15T22:14:34,575 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f2783381a48c13c2f75c07e5029903cb, ASSIGN; state=OFFLINE, location=e611192d6313,43349,1731708869385; forceNewPlan=false, retain=false 2024-11-15T22:14:34,588 INFO [e611192d6313:39087 {}] balancer.BaseLoadBalancer(388): Reassigned 26 regions. 26 retained the pre-restart assignment. 2024-11-15T22:14:34,590 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=2e79a8ce6f9715045d056d3dfdef5cec, regionState=OPENING, regionLocation=e611192d6313,43349,1731708869385 2024-11-15T22:14:34,591 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=819421a6406c0ebb5635c399b2c0303d, regionState=OPENING, regionLocation=e611192d6313,43349,1731708869385 2024-11-15T22:14:34,592 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=06b9df8f55d791077e9f7ec55ef6a971, regionState=OPENING, regionLocation=e611192d6313,43349,1731708869385 2024-11-15T22:14:34,593 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=c56d152c07a95a189ac65341ff4db1fa, regionState=OPENING, regionLocation=e611192d6313,43349,1731708869385 2024-11-15T22:14:34,594 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=2a159d388ea2138ee8e2bed061a785ce, regionState=OPENING, regionLocation=e611192d6313,43349,1731708869385 2024-11-15T22:14:34,595 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=13c3457b9645c9a8181ed9b4551962ad, regionState=OPENING, regionLocation=e611192d6313,43349,1731708869385 2024-11-15T22:14:34,597 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=93deaad3e3229244196e8daa9478371a, regionState=OPENING, regionLocation=e611192d6313,43349,1731708869385 2024-11-15T22:14:34,598 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=897a7bb57a002d365489ce583eb7f84f, regionState=OPENING, regionLocation=e611192d6313,43349,1731708869385 2024-11-15T22:14:34,599 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=f2783381a48c13c2f75c07e5029903cb, regionState=OPENING, regionLocation=e611192d6313,43349,1731708869385 2024-11-15T22:14:34,599 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=a7fb991e3993a690fe7a0a1b064d5f5d, regionState=OPENING, regionLocation=e611192d6313,41231,1731708869041 2024-11-15T22:14:34,600 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=e87a82b283b7251342a664ba3876c98e, regionState=OPENING, regionLocation=e611192d6313,41231,1731708869041 2024-11-15T22:14:34,601 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=4381d14830a288bee3aeb42195dd8768, regionState=OPENING, regionLocation=e611192d6313,41231,1731708869041 2024-11-15T22:14:34,602 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=83048e0be415b190a19f64bc2e49d3fe, regionState=OPENING, regionLocation=e611192d6313,41231,1731708869041 2024-11-15T22:14:34,604 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=f64d8a749084ba25ea1b77db9a0d9f17, regionState=OPENING, regionLocation=e611192d6313,41231,1731708869041 2024-11-15T22:14:34,605 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=36b8b3bed3adb94f5901f1f3809b64fa, regionState=OPENING, regionLocation=e611192d6313,41231,1731708869041 2024-11-15T22:14:34,606 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=1959cb95bb21ecdc677f7f5290173984, regionState=OPENING, regionLocation=e611192d6313,41231,1731708869041 2024-11-15T22:14:34,607 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=51bf57bcf1afa9f625176363284d17f7, regionState=OPENING, regionLocation=e611192d6313,41231,1731708869041 2024-11-15T22:14:34,608 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=6efdb15046141ab0cd574540ef2ac2c2, regionState=OPENING, regionLocation=e611192d6313,38729,1731708868111 2024-11-15T22:14:34,609 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=2e79a8ce6f9715045d056d3dfdef5cec, ASSIGN because future has completed 2024-11-15T22:14:34,611 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2e79a8ce6f9715045d056d3dfdef5cec, server=e611192d6313,43349,1731708869385}] 2024-11-15T22:14:34,615 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN because future has completed 2024-11-15T22:14:34,619 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=06b9df8f55d791077e9f7ec55ef6a971, ASSIGN because future has completed 2024-11-15T22:14:34,624 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c56d152c07a95a189ac65341ff4db1fa, ASSIGN because future has completed 2024-11-15T22:14:34,625 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06b9df8f55d791077e9f7ec55ef6a971, server=e611192d6313,43349,1731708869385}] 2024-11-15T22:14:34,629 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN because future has completed 2024-11-15T22:14:34,634 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN because future has completed 2024-11-15T22:14:34,639 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=93deaad3e3229244196e8daa9478371a, ASSIGN because future has completed 2024-11-15T22:14:34,641 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 93deaad3e3229244196e8daa9478371a, server=e611192d6313,43349,1731708869385}] 2024-11-15T22:14:34,642 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=897a7bb57a002d365489ce583eb7f84f, ASSIGN because future has completed 2024-11-15T22:14:34,648 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f2783381a48c13c2f75c07e5029903cb, ASSIGN because future has completed 2024-11-15T22:14:34,652 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a7fb991e3993a690fe7a0a1b064d5f5d, ASSIGN because future has completed 2024-11-15T22:14:34,657 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=e87a82b283b7251342a664ba3876c98e, ASSIGN because future has completed 2024-11-15T22:14:34,659 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure e87a82b283b7251342a664ba3876c98e, server=e611192d6313,41231,1731708869041}] 2024-11-15T22:14:34,660 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4381d14830a288bee3aeb42195dd8768, ASSIGN because future has completed 2024-11-15T22:14:34,663 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=83048e0be415b190a19f64bc2e49d3fe, ASSIGN because future has completed 2024-11-15T22:14:34,666 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f64d8a749084ba25ea1b77db9a0d9f17, ASSIGN because future has completed 2024-11-15T22:14:34,670 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=36b8b3bed3adb94f5901f1f3809b64fa, ASSIGN because future has completed 2024-11-15T22:14:34,672 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=1959cb95bb21ecdc677f7f5290173984, ASSIGN because future has completed 2024-11-15T22:14:34,676 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1959cb95bb21ecdc677f7f5290173984, server=e611192d6313,41231,1731708869041}] 2024-11-15T22:14:34,677 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=51bf57bcf1afa9f625176363284d17f7, ASSIGN because future has completed 2024-11-15T22:14:34,681 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=6efdb15046141ab0cd574540ef2ac2c2, ASSIGN because future has completed 2024-11-15T22:14:34,688 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6efdb15046141ab0cd574540ef2ac2c2, server=e611192d6313,38729,1731708868111}] 2024-11-15T22:14:34,699 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 51bf57bcf1afa9f625176363284d17f7, server=e611192d6313,41231,1731708869041}] 2024-11-15T22:14:34,708 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 36b8b3bed3adb94f5901f1f3809b64fa, server=e611192d6313,41231,1731708869041}] 2024-11-15T22:14:34,717 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure f64d8a749084ba25ea1b77db9a0d9f17, server=e611192d6313,41231,1731708869041}] 2024-11-15T22:14:34,724 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 83048e0be415b190a19f64bc2e49d3fe, server=e611192d6313,41231,1731708869041}] 2024-11-15T22:14:34,747 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4381d14830a288bee3aeb42195dd8768, server=e611192d6313,41231,1731708869041}] 2024-11-15T22:14:34,760 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure a7fb991e3993a690fe7a0a1b064d5f5d, server=e611192d6313,41231,1731708869041}] 2024-11-15T22:14:34,774 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure f2783381a48c13c2f75c07e5029903cb, server=e611192d6313,43349,1731708869385}] 2024-11-15T22:14:34,788 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open test,www,1731708873681.2e79a8ce6f9715045d056d3dfdef5cec. 2024-11-15T22:14:34,789 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => 2e79a8ce6f9715045d056d3dfdef5cec, NAME => 'test,www,1731708873681.2e79a8ce6f9715045d056d3dfdef5cec.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-11-15T22:14:34,789 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 2e79a8ce6f9715045d056d3dfdef5cec 2024-11-15T22:14:34,789 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated test,www,1731708873681.2e79a8ce6f9715045d056d3dfdef5cec.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:34,789 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for 2e79a8ce6f9715045d056d3dfdef5cec 2024-11-15T22:14:34,790 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for 2e79a8ce6f9715045d056d3dfdef5cec 2024-11-15T22:14:34,797 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 897a7bb57a002d365489ce583eb7f84f, server=e611192d6313,43349,1731708869385}] 2024-11-15T22:14:34,807 INFO [StoreOpener-2e79a8ce6f9715045d056d3dfdef5cec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 2e79a8ce6f9715045d056d3dfdef5cec 2024-11-15T22:14:34,815 INFO [StoreOpener-2e79a8ce6f9715045d056d3dfdef5cec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2e79a8ce6f9715045d056d3dfdef5cec columnFamilyName family 2024-11-15T22:14:34,815 DEBUG [StoreOpener-2e79a8ce6f9715045d056d3dfdef5cec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:34,817 INFO [StoreOpener-2e79a8ce6f9715045d056d3dfdef5cec-1 {}] regionserver.HStore(327): Store=2e79a8ce6f9715045d056d3dfdef5cec/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:14:34,817 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for 2e79a8ce6f9715045d056d3dfdef5cec 2024-11-15T22:14:34,818 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/2e79a8ce6f9715045d056d3dfdef5cec 2024-11-15T22:14:34,821 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/2e79a8ce6f9715045d056d3dfdef5cec 2024-11-15T22:14:34,825 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-15T22:14:34,825 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for 2e79a8ce6f9715045d056d3dfdef5cec 2024-11-15T22:14:34,825 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for 2e79a8ce6f9715045d056d3dfdef5cec 2024-11-15T22:14:34,832 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for 2e79a8ce6f9715045d056d3dfdef5cec 2024-11-15T22:14:34,833 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50907, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-15T22:14:34,838 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,43349,1731708869385}] 2024-11-15T22:14:34,845 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-15T22:14:34,848 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open test,ttt,1731708873681.4381d14830a288bee3aeb42195dd8768. 2024-11-15T22:14:34,848 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => 4381d14830a288bee3aeb42195dd8768, NAME => 'test,ttt,1731708873681.4381d14830a288bee3aeb42195dd8768.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-15T22:14:34,848 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/2e79a8ce6f9715045d056d3dfdef5cec/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T22:14:34,849 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4381d14830a288bee3aeb42195dd8768 2024-11-15T22:14:34,849 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated test,ttt,1731708873681.4381d14830a288bee3aeb42195dd8768.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:34,849 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for 4381d14830a288bee3aeb42195dd8768 2024-11-15T22:14:34,849 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for 4381d14830a288bee3aeb42195dd8768 2024-11-15T22:14:34,854 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened 2e79a8ce6f9715045d056d3dfdef5cec; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74873154, jitterRate=0.11569693684577942}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T22:14:34,854 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2e79a8ce6f9715045d056d3dfdef5cec 2024-11-15T22:14:34,855 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for 2e79a8ce6f9715045d056d3dfdef5cec: Running coprocessor pre-open hook at 1731708874790Writing region info on filesystem at 1731708874790Initializing all the Stores at 1731708874800 (+10 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708874800Cleaning up temporary data from old regions at 1731708874825 (+25 ms)Running coprocessor post-open hooks at 1731708874854 (+29 ms)Region opened successfully at 1731708874855 (+1 ms) 2024-11-15T22:14:34,857 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45117, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-15T22:14:34,862 INFO [StoreOpener-4381d14830a288bee3aeb42195dd8768-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4381d14830a288bee3aeb42195dd8768 2024-11-15T22:14:34,869 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for test,www,1731708873681.2e79a8ce6f9715045d056d3dfdef5cec., pid=31, masterSystemTime=1731708874768 2024-11-15T22:14:34,871 INFO [StoreOpener-4381d14830a288bee3aeb42195dd8768-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4381d14830a288bee3aeb42195dd8768 columnFamilyName family 2024-11-15T22:14:34,871 DEBUG [StoreOpener-4381d14830a288bee3aeb42195dd8768-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:34,872 INFO [StoreOpener-4381d14830a288bee3aeb42195dd8768-1 {}] regionserver.HStore(327): Store=4381d14830a288bee3aeb42195dd8768/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:14:34,872 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for 4381d14830a288bee3aeb42195dd8768 2024-11-15T22:14:34,874 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2a159d388ea2138ee8e2bed061a785ce, server=e611192d6313,43349,1731708869385}] 2024-11-15T22:14:34,876 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/4381d14830a288bee3aeb42195dd8768 2024-11-15T22:14:34,877 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for test,www,1731708873681.2e79a8ce6f9715045d056d3dfdef5cec. 2024-11-15T22:14:34,878 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened test,www,1731708873681.2e79a8ce6f9715045d056d3dfdef5cec. 2024-11-15T22:14:34,878 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open test,qqq,1731708873681.06b9df8f55d791077e9f7ec55ef6a971. 2024-11-15T22:14:34,878 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 06b9df8f55d791077e9f7ec55ef6a971, NAME => 'test,qqq,1731708873681.06b9df8f55d791077e9f7ec55ef6a971.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-11-15T22:14:34,879 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 06b9df8f55d791077e9f7ec55ef6a971 2024-11-15T22:14:34,879 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated test,qqq,1731708873681.06b9df8f55d791077e9f7ec55ef6a971.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:34,879 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 06b9df8f55d791077e9f7ec55ef6a971 2024-11-15T22:14:34,879 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 06b9df8f55d791077e9f7ec55ef6a971 2024-11-15T22:14:34,883 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/4381d14830a288bee3aeb42195dd8768 2024-11-15T22:14:34,886 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open test,zzz,1731708873681.6efdb15046141ab0cd574540ef2ac2c2. 2024-11-15T22:14:34,886 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for 4381d14830a288bee3aeb42195dd8768 2024-11-15T22:14:34,887 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for 4381d14830a288bee3aeb42195dd8768 2024-11-15T22:14:34,888 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => 6efdb15046141ab0cd574540ef2ac2c2, NAME => 'test,zzz,1731708873681.6efdb15046141ab0cd574540ef2ac2c2.', STARTKEY => 'zzz', ENDKEY => ''} 2024-11-15T22:14:34,888 INFO [StoreOpener-06b9df8f55d791077e9f7ec55ef6a971-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 06b9df8f55d791077e9f7ec55ef6a971 2024-11-15T22:14:34,889 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 6efdb15046141ab0cd574540ef2ac2c2 2024-11-15T22:14:34,894 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated test,zzz,1731708873681.6efdb15046141ab0cd574540ef2ac2c2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:34,895 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=2e79a8ce6f9715045d056d3dfdef5cec, regionState=OPEN, openSeqNum=2, regionLocation=e611192d6313,43349,1731708869385 2024-11-15T22:14:34,894 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for 6efdb15046141ab0cd574540ef2ac2c2 2024-11-15T22:14:34,895 INFO [StoreOpener-06b9df8f55d791077e9f7ec55ef6a971-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06b9df8f55d791077e9f7ec55ef6a971 columnFamilyName family 2024-11-15T22:14:34,895 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for 6efdb15046141ab0cd574540ef2ac2c2 2024-11-15T22:14:34,895 DEBUG [StoreOpener-06b9df8f55d791077e9f7ec55ef6a971-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:34,895 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for 4381d14830a288bee3aeb42195dd8768 2024-11-15T22:14:34,896 INFO [StoreOpener-06b9df8f55d791077e9f7ec55ef6a971-1 {}] regionserver.HStore(327): Store=06b9df8f55d791077e9f7ec55ef6a971/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:14:34,896 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 06b9df8f55d791077e9f7ec55ef6a971 2024-11-15T22:14:34,898 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/06b9df8f55d791077e9f7ec55ef6a971 2024-11-15T22:14:34,899 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/06b9df8f55d791077e9f7ec55ef6a971 2024-11-15T22:14:34,901 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure c56d152c07a95a189ac65341ff4db1fa, server=e611192d6313,43349,1731708869385}] 2024-11-15T22:14:34,901 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 06b9df8f55d791077e9f7ec55ef6a971 2024-11-15T22:14:34,901 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 06b9df8f55d791077e9f7ec55ef6a971 2024-11-15T22:14:34,903 INFO [StoreOpener-6efdb15046141ab0cd574540ef2ac2c2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 6efdb15046141ab0cd574540ef2ac2c2 2024-11-15T22:14:34,912 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/4381d14830a288bee3aeb42195dd8768/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T22:14:34,913 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2e79a8ce6f9715045d056d3dfdef5cec, server=e611192d6313,43349,1731708869385 because future has completed 2024-11-15T22:14:34,913 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened 4381d14830a288bee3aeb42195dd8768; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70579227, jitterRate=0.051712438464164734}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T22:14:34,913 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4381d14830a288bee3aeb42195dd8768 2024-11-15T22:14:34,914 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for 4381d14830a288bee3aeb42195dd8768: Running coprocessor pre-open hook at 1731708874849Writing region info on filesystem at 1731708874849Initializing all the Stores at 1731708874862 (+13 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708874862Cleaning up temporary data from old regions at 1731708874887 (+25 ms)Running coprocessor post-open hooks at 1731708874913 (+26 ms)Region opened successfully at 1731708874914 (+1 ms) 2024-11-15T22:14:34,914 INFO [StoreOpener-6efdb15046141ab0cd574540ef2ac2c2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6efdb15046141ab0cd574540ef2ac2c2 columnFamilyName family 2024-11-15T22:14:34,916 DEBUG [StoreOpener-6efdb15046141ab0cd574540ef2ac2c2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:34,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T22:14:34,919 INFO [StoreOpener-6efdb15046141ab0cd574540ef2ac2c2-1 {}] regionserver.HStore(327): Store=6efdb15046141ab0cd574540ef2ac2c2/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:14:34,919 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for 6efdb15046141ab0cd574540ef2ac2c2 2024-11-15T22:14:34,920 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 06b9df8f55d791077e9f7ec55ef6a971 2024-11-15T22:14:34,920 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/6efdb15046141ab0cd574540ef2ac2c2 2024-11-15T22:14:34,922 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/6efdb15046141ab0cd574540ef2ac2c2 2024-11-15T22:14:34,923 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for 6efdb15046141ab0cd574540ef2ac2c2 2024-11-15T22:14:34,923 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for 6efdb15046141ab0cd574540ef2ac2c2 2024-11-15T22:14:34,924 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1731708873681.4381d14830a288bee3aeb42195dd8768., pid=41, masterSystemTime=1731708874824 2024-11-15T22:14:34,927 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for 6efdb15046141ab0cd574540ef2ac2c2 2024-11-15T22:14:34,944 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1731708873681.4381d14830a288bee3aeb42195dd8768. 2024-11-15T22:14:34,945 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened test,ttt,1731708873681.4381d14830a288bee3aeb42195dd8768. 2024-11-15T22:14:34,945 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open test,mmm,1731708873681.f64d8a749084ba25ea1b77db9a0d9f17. 2024-11-15T22:14:34,945 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => f64d8a749084ba25ea1b77db9a0d9f17, NAME => 'test,mmm,1731708873681.f64d8a749084ba25ea1b77db9a0d9f17.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-15T22:14:34,946 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f64d8a749084ba25ea1b77db9a0d9f17 2024-11-15T22:14:34,946 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated test,mmm,1731708873681.f64d8a749084ba25ea1b77db9a0d9f17.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:34,946 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for f64d8a749084ba25ea1b77db9a0d9f17 2024-11-15T22:14:34,946 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for f64d8a749084ba25ea1b77db9a0d9f17 2024-11-15T22:14:34,947 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/6efdb15046141ab0cd574540ef2ac2c2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T22:14:34,947 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/06b9df8f55d791077e9f7ec55ef6a971/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T22:14:34,949 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened 6efdb15046141ab0cd574540ef2ac2c2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74560043, jitterRate=0.11103121936321259}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T22:14:34,949 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6efdb15046141ab0cd574540ef2ac2c2 2024-11-15T22:14:34,950 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for 6efdb15046141ab0cd574540ef2ac2c2: Running coprocessor pre-open hook at 1731708874895Writing region info on filesystem at 1731708874895Initializing all the Stores at 1731708874902 (+7 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708874902Cleaning up temporary data from old regions at 1731708874923 (+21 ms)Running coprocessor post-open hooks at 1731708874949 (+26 ms)Region opened successfully at 1731708874949 2024-11-15T22:14:34,953 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for test,zzz,1731708873681.6efdb15046141ab0cd574540ef2ac2c2., pid=36, masterSystemTime=1731708874845 2024-11-15T22:14:34,955 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 06b9df8f55d791077e9f7ec55ef6a971; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69935590, jitterRate=0.04212149977684021}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T22:14:34,955 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 06b9df8f55d791077e9f7ec55ef6a971 2024-11-15T22:14:34,955 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 06b9df8f55d791077e9f7ec55ef6a971: Running coprocessor pre-open hook at 1731708874879Writing region info on filesystem at 1731708874879Initializing all the Stores at 1731708874887 (+8 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708874887Cleaning up temporary data from old regions at 1731708874901 (+14 ms)Running coprocessor post-open hooks at 1731708874955 (+54 ms)Region opened successfully at 1731708874955 2024-11-15T22:14:34,956 INFO [StoreOpener-f64d8a749084ba25ea1b77db9a0d9f17-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f64d8a749084ba25ea1b77db9a0d9f17 2024-11-15T22:14:34,961 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for test,qqq,1731708873681.06b9df8f55d791077e9f7ec55ef6a971., pid=32, masterSystemTime=1731708874768 2024-11-15T22:14:34,965 INFO [StoreOpener-f64d8a749084ba25ea1b77db9a0d9f17-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f64d8a749084ba25ea1b77db9a0d9f17 columnFamilyName family 2024-11-15T22:14:34,965 DEBUG [StoreOpener-f64d8a749084ba25ea1b77db9a0d9f17-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:34,966 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for test,zzz,1731708873681.6efdb15046141ab0cd574540ef2ac2c2. 2024-11-15T22:14:34,966 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened test,zzz,1731708873681.6efdb15046141ab0cd574540ef2ac2c2. 2024-11-15T22:14:34,966 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=27 2024-11-15T22:14:34,966 INFO [StoreOpener-f64d8a749084ba25ea1b77db9a0d9f17-1 {}] regionserver.HStore(327): Store=f64d8a749084ba25ea1b77db9a0d9f17/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:14:34,967 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure 2e79a8ce6f9715045d056d3dfdef5cec, server=e611192d6313,43349,1731708869385 in 338 msec 2024-11-15T22:14:34,967 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for f64d8a749084ba25ea1b77db9a0d9f17 2024-11-15T22:14:34,968 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/f64d8a749084ba25ea1b77db9a0d9f17 2024-11-15T22:14:34,971 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/f64d8a749084ba25ea1b77db9a0d9f17 2024-11-15T22:14:34,973 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for f64d8a749084ba25ea1b77db9a0d9f17 2024-11-15T22:14:34,974 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for f64d8a749084ba25ea1b77db9a0d9f17 2024-11-15T22:14:34,977 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for test,qqq,1731708873681.06b9df8f55d791077e9f7ec55ef6a971. 2024-11-15T22:14:34,977 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened test,qqq,1731708873681.06b9df8f55d791077e9f7ec55ef6a971. 2024-11-15T22:14:34,977 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open test,eee,1731708873681.93deaad3e3229244196e8daa9478371a. 2024-11-15T22:14:34,977 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 93deaad3e3229244196e8daa9478371a, NAME => 'test,eee,1731708873681.93deaad3e3229244196e8daa9478371a.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-15T22:14:34,978 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 93deaad3e3229244196e8daa9478371a 2024-11-15T22:14:34,979 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated test,eee,1731708873681.93deaad3e3229244196e8daa9478371a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:34,979 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 93deaad3e3229244196e8daa9478371a 2024-11-15T22:14:34,979 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 93deaad3e3229244196e8daa9478371a 2024-11-15T22:14:34,981 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=2e79a8ce6f9715045d056d3dfdef5cec, ASSIGN in 547 msec 2024-11-15T22:14:34,988 INFO [StoreOpener-93deaad3e3229244196e8daa9478371a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 93deaad3e3229244196e8daa9478371a 2024-11-15T22:14:34,989 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for f64d8a749084ba25ea1b77db9a0d9f17 2024-11-15T22:14:34,990 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=06b9df8f55d791077e9f7ec55ef6a971, regionState=OPEN, openSeqNum=2, regionLocation=e611192d6313,43349,1731708869385 2024-11-15T22:14:34,997 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39087 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=e611192d6313,43349,1731708869385, table=test, region=06b9df8f55d791077e9f7ec55ef6a971. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T22:14:34,997 INFO [StoreOpener-93deaad3e3229244196e8daa9478371a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 93deaad3e3229244196e8daa9478371a columnFamilyName family 2024-11-15T22:14:34,997 DEBUG [StoreOpener-93deaad3e3229244196e8daa9478371a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:34,998 INFO [StoreOpener-93deaad3e3229244196e8daa9478371a-1 {}] regionserver.HStore(327): Store=93deaad3e3229244196e8daa9478371a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:14:34,999 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 93deaad3e3229244196e8daa9478371a 2024-11-15T22:14:35,000 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/f64d8a749084ba25ea1b77db9a0d9f17/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T22:14:35,002 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=6efdb15046141ab0cd574540ef2ac2c2, regionState=OPEN, openSeqNum=2, regionLocation=e611192d6313,38729,1731708868111 2024-11-15T22:14:35,004 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06b9df8f55d791077e9f7ec55ef6a971, server=e611192d6313,43349,1731708869385 because future has completed 2024-11-15T22:14:35,004 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/93deaad3e3229244196e8daa9478371a 2024-11-15T22:14:35,004 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened f64d8a749084ba25ea1b77db9a0d9f17; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73032825, jitterRate=0.08827389776706696}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T22:14:35,004 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f64d8a749084ba25ea1b77db9a0d9f17 2024-11-15T22:14:35,004 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for f64d8a749084ba25ea1b77db9a0d9f17: Running coprocessor pre-open hook at 1731708874946Writing region info on filesystem at 1731708874946Initializing all the Stores at 1731708874951 (+5 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708874951Cleaning up temporary data from old regions at 1731708874974 (+23 ms)Running coprocessor post-open hooks at 1731708875004 (+30 ms)Region opened successfully at 1731708875004 2024-11-15T22:14:35,007 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/93deaad3e3229244196e8daa9478371a 2024-11-15T22:14:35,011 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 93deaad3e3229244196e8daa9478371a 2024-11-15T22:14:35,011 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 93deaad3e3229244196e8daa9478371a 2024-11-15T22:14:35,013 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1731708873681.f64d8a749084ba25ea1b77db9a0d9f17., pid=39, masterSystemTime=1731708874824 2024-11-15T22:14:35,014 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6efdb15046141ab0cd574540ef2ac2c2, server=e611192d6313,38729,1731708868111 because future has completed 2024-11-15T22:14:35,025 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1731708873681.f64d8a749084ba25ea1b77db9a0d9f17. 2024-11-15T22:14:35,025 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened test,mmm,1731708873681.f64d8a749084ba25ea1b77db9a0d9f17. 2024-11-15T22:14:35,025 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(132): Open test,xxx,1731708873681.a7fb991e3993a690fe7a0a1b064d5f5d. 2024-11-15T22:14:35,026 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7752): Opening region: {ENCODED => a7fb991e3993a690fe7a0a1b064d5f5d, NAME => 'test,xxx,1731708873681.a7fb991e3993a690fe7a0a1b064d5f5d.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-11-15T22:14:35,026 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test a7fb991e3993a690fe7a0a1b064d5f5d 2024-11-15T22:14:35,026 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(898): Instantiated test,xxx,1731708873681.a7fb991e3993a690fe7a0a1b064d5f5d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:35,026 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7794): checking encryption for a7fb991e3993a690fe7a0a1b064d5f5d 2024-11-15T22:14:35,026 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7797): checking classloading for a7fb991e3993a690fe7a0a1b064d5f5d 2024-11-15T22:14:35,032 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 93deaad3e3229244196e8daa9478371a 2024-11-15T22:14:35,036 INFO [StoreOpener-a7fb991e3993a690fe7a0a1b064d5f5d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region a7fb991e3993a690fe7a0a1b064d5f5d 2024-11-15T22:14:35,041 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=21 2024-11-15T22:14:35,041 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=21, state=SUCCESS, hasLock=false; OpenRegionProcedure 06b9df8f55d791077e9f7ec55ef6a971, server=e611192d6313,43349,1731708869385 in 389 msec 2024-11-15T22:14:35,043 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/93deaad3e3229244196e8daa9478371a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T22:14:35,044 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 93deaad3e3229244196e8daa9478371a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69209641, jitterRate=0.0313040167093277}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T22:14:35,044 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 93deaad3e3229244196e8daa9478371a 2024-11-15T22:14:35,045 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 93deaad3e3229244196e8daa9478371a: Running coprocessor pre-open hook at 1731708874980Writing region info on filesystem at 1731708874980Initializing all the Stores at 1731708874982 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708874983 (+1 ms)Cleaning up temporary data from old regions at 1731708875011 (+28 ms)Running coprocessor post-open hooks at 1731708875044 (+33 ms)Region opened successfully at 1731708875045 (+1 ms) 2024-11-15T22:14:35,045 INFO [StoreOpener-a7fb991e3993a690fe7a0a1b064d5f5d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a7fb991e3993a690fe7a0a1b064d5f5d columnFamilyName family 2024-11-15T22:14:35,045 DEBUG [StoreOpener-a7fb991e3993a690fe7a0a1b064d5f5d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:35,047 INFO [StoreOpener-a7fb991e3993a690fe7a0a1b064d5f5d-1 {}] regionserver.HStore(327): Store=a7fb991e3993a690fe7a0a1b064d5f5d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:14:35,047 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1038): replaying wal for a7fb991e3993a690fe7a0a1b064d5f5d 2024-11-15T22:14:35,053 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1731708873681.93deaad3e3229244196e8daa9478371a., pid=33, masterSystemTime=1731708874768 2024-11-15T22:14:35,053 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/a7fb991e3993a690fe7a0a1b064d5f5d 2024-11-15T22:14:35,054 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/a7fb991e3993a690fe7a0a1b064d5f5d 2024-11-15T22:14:35,056 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=06b9df8f55d791077e9f7ec55ef6a971, ASSIGN in 621 msec 2024-11-15T22:14:35,059 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1048): stopping wal replay for a7fb991e3993a690fe7a0a1b064d5f5d 2024-11-15T22:14:35,059 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1060): Cleaning up temporary data for a7fb991e3993a690fe7a0a1b064d5f5d 2024-11-15T22:14:35,061 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=f64d8a749084ba25ea1b77db9a0d9f17, regionState=OPEN, openSeqNum=2, regionLocation=e611192d6313,41231,1731708869041 2024-11-15T22:14:35,066 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1731708873681.93deaad3e3229244196e8daa9478371a. 2024-11-15T22:14:35,066 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened test,eee,1731708873681.93deaad3e3229244196e8daa9478371a. 2024-11-15T22:14:35,067 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open test,kkk,1731708873681.2a159d388ea2138ee8e2bed061a785ce. 2024-11-15T22:14:35,067 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1093): writing seq id for a7fb991e3993a690fe7a0a1b064d5f5d 2024-11-15T22:14:35,067 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => 2a159d388ea2138ee8e2bed061a785ce, NAME => 'test,kkk,1731708873681.2a159d388ea2138ee8e2bed061a785ce.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-11-15T22:14:35,067 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 2a159d388ea2138ee8e2bed061a785ce 2024-11-15T22:14:35,067 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated test,kkk,1731708873681.2a159d388ea2138ee8e2bed061a785ce.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:35,068 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for 2a159d388ea2138ee8e2bed061a785ce 2024-11-15T22:14:35,068 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for 2a159d388ea2138ee8e2bed061a785ce 2024-11-15T22:14:35,068 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39087 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=e611192d6313,38729,1731708868111, table=test, region=6efdb15046141ab0cd574540ef2ac2c2. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T22:14:35,072 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=93deaad3e3229244196e8daa9478371a, regionState=OPEN, openSeqNum=2, regionLocation=e611192d6313,43349,1731708869385 2024-11-15T22:14:35,076 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure f64d8a749084ba25ea1b77db9a0d9f17, server=e611192d6313,41231,1731708869041 because future has completed 2024-11-15T22:14:35,081 INFO [StoreOpener-2a159d388ea2138ee8e2bed061a785ce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 2a159d388ea2138ee8e2bed061a785ce 2024-11-15T22:14:35,082 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/a7fb991e3993a690fe7a0a1b064d5f5d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T22:14:35,084 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1114): Opened a7fb991e3993a690fe7a0a1b064d5f5d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61751105, jitterRate=-0.07983683049678802}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T22:14:35,085 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a7fb991e3993a690fe7a0a1b064d5f5d 2024-11-15T22:14:35,085 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1006): Region open journal for a7fb991e3993a690fe7a0a1b064d5f5d: Running coprocessor pre-open hook at 1731708875027Writing region info on filesystem at 1731708875027Initializing all the Stores at 1731708875034 (+7 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708875034Cleaning up temporary data from old regions at 1731708875059 (+25 ms)Running coprocessor post-open hooks at 1731708875085 (+26 ms)Region opened successfully at 1731708875085 2024-11-15T22:14:35,085 INFO [StoreOpener-2a159d388ea2138ee8e2bed061a785ce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2a159d388ea2138ee8e2bed061a785ce columnFamilyName family 2024-11-15T22:14:35,085 DEBUG [StoreOpener-2a159d388ea2138ee8e2bed061a785ce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:35,086 INFO [StoreOpener-2a159d388ea2138ee8e2bed061a785ce-1 {}] regionserver.HStore(327): Store=2a159d388ea2138ee8e2bed061a785ce/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:14:35,086 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for 2a159d388ea2138ee8e2bed061a785ce 2024-11-15T22:14:35,088 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/2a159d388ea2138ee8e2bed061a785ce 2024-11-15T22:14:35,089 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/2a159d388ea2138ee8e2bed061a785ce 2024-11-15T22:14:35,091 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for 2a159d388ea2138ee8e2bed061a785ce 2024-11-15T22:14:35,091 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for 2a159d388ea2138ee8e2bed061a785ce 2024-11-15T22:14:35,092 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1731708873681.a7fb991e3993a690fe7a0a1b064d5f5d., pid=42, masterSystemTime=1731708874824 2024-11-15T22:14:35,093 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 93deaad3e3229244196e8daa9478371a, server=e611192d6313,43349,1731708869385 because future has completed 2024-11-15T22:14:35,095 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for 2a159d388ea2138ee8e2bed061a785ce 2024-11-15T22:14:35,098 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1731708873681.a7fb991e3993a690fe7a0a1b064d5f5d. 2024-11-15T22:14:35,099 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(153): Opened test,xxx,1731708873681.a7fb991e3993a690fe7a0a1b064d5f5d. 2024-11-15T22:14:35,099 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open test,ddd,1731708873681.51bf57bcf1afa9f625176363284d17f7. 2024-11-15T22:14:35,099 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 51bf57bcf1afa9f625176363284d17f7, NAME => 'test,ddd,1731708873681.51bf57bcf1afa9f625176363284d17f7.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-11-15T22:14:35,100 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=30 2024-11-15T22:14:35,100 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure 6efdb15046141ab0cd574540ef2ac2c2, server=e611192d6313,38729,1731708868111 in 394 msec 2024-11-15T22:14:35,101 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 51bf57bcf1afa9f625176363284d17f7 2024-11-15T22:14:35,101 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated test,ddd,1731708873681.51bf57bcf1afa9f625176363284d17f7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:35,101 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 51bf57bcf1afa9f625176363284d17f7 2024-11-15T22:14:35,101 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 51bf57bcf1afa9f625176363284d17f7 2024-11-15T22:14:35,103 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39087 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=e611192d6313,43349,1731708869385, table=test, region=93deaad3e3229244196e8daa9478371a. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T22:14:35,104 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/2a159d388ea2138ee8e2bed061a785ce/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T22:14:35,105 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened 2a159d388ea2138ee8e2bed061a785ce; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71918645, jitterRate=0.07167132198810577}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T22:14:35,105 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2a159d388ea2138ee8e2bed061a785ce 2024-11-15T22:14:35,105 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for 2a159d388ea2138ee8e2bed061a785ce: Running coprocessor pre-open hook at 1731708875068Writing region info on filesystem at 1731708875068Initializing all the Stores at 1731708875076 (+8 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708875076Cleaning up temporary data from old regions at 1731708875091 (+15 ms)Running coprocessor post-open hooks at 1731708875105 (+14 ms)Region opened successfully at 1731708875105 2024-11-15T22:14:35,107 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6efdb15046141ab0cd574540ef2ac2c2, ASSIGN in 680 msec 2024-11-15T22:14:35,106 INFO [StoreOpener-51bf57bcf1afa9f625176363284d17f7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 51bf57bcf1afa9f625176363284d17f7 2024-11-15T22:14:35,109 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1731708873681.2a159d388ea2138ee8e2bed061a785ce., pid=46, masterSystemTime=1731708874940 2024-11-15T22:14:35,112 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=a7fb991e3993a690fe7a0a1b064d5f5d, regionState=OPEN, openSeqNum=2, regionLocation=e611192d6313,41231,1731708869041 2024-11-15T22:14:35,116 INFO [StoreOpener-51bf57bcf1afa9f625176363284d17f7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 51bf57bcf1afa9f625176363284d17f7 columnFamilyName family 2024-11-15T22:14:35,116 DEBUG [StoreOpener-51bf57bcf1afa9f625176363284d17f7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:35,117 INFO [StoreOpener-51bf57bcf1afa9f625176363284d17f7-1 {}] regionserver.HStore(327): Store=51bf57bcf1afa9f625176363284d17f7/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:14:35,117 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 51bf57bcf1afa9f625176363284d17f7 2024-11-15T22:14:35,118 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure a7fb991e3993a690fe7a0a1b064d5f5d, server=e611192d6313,41231,1731708869041 because future has completed 2024-11-15T22:14:35,119 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/51bf57bcf1afa9f625176363284d17f7 2024-11-15T22:14:35,120 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/51bf57bcf1afa9f625176363284d17f7 2024-11-15T22:14:35,121 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for test,kkk,1731708873681.2a159d388ea2138ee8e2bed061a785ce. 2024-11-15T22:14:35,121 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened test,kkk,1731708873681.2a159d388ea2138ee8e2bed061a785ce. 2024-11-15T22:14:35,121 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open test,hhh,1731708873681.13c3457b9645c9a8181ed9b4551962ad. 2024-11-15T22:14:35,121 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => 13c3457b9645c9a8181ed9b4551962ad, NAME => 'test,hhh,1731708873681.13c3457b9645c9a8181ed9b4551962ad.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-15T22:14:35,122 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 13c3457b9645c9a8181ed9b4551962ad 2024-11-15T22:14:35,122 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated test,hhh,1731708873681.13c3457b9645c9a8181ed9b4551962ad.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:35,122 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for 13c3457b9645c9a8181ed9b4551962ad 2024-11-15T22:14:35,122 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for 13c3457b9645c9a8181ed9b4551962ad 2024-11-15T22:14:35,124 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 51bf57bcf1afa9f625176363284d17f7 2024-11-15T22:14:35,125 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 51bf57bcf1afa9f625176363284d17f7 2024-11-15T22:14:35,125 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=2a159d388ea2138ee8e2bed061a785ce, regionState=OPEN, openSeqNum=2, regionLocation=e611192d6313,43349,1731708869385 2024-11-15T22:14:35,126 INFO [StoreOpener-13c3457b9645c9a8181ed9b4551962ad-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 13c3457b9645c9a8181ed9b4551962ad 2024-11-15T22:14:35,128 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 51bf57bcf1afa9f625176363284d17f7 2024-11-15T22:14:35,130 INFO [StoreOpener-13c3457b9645c9a8181ed9b4551962ad-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 13c3457b9645c9a8181ed9b4551962ad columnFamilyName family 2024-11-15T22:14:35,130 DEBUG [StoreOpener-13c3457b9645c9a8181ed9b4551962ad-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:35,132 INFO [StoreOpener-13c3457b9645c9a8181ed9b4551962ad-1 {}] regionserver.HStore(327): Store=13c3457b9645c9a8181ed9b4551962ad/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:14:35,133 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for 13c3457b9645c9a8181ed9b4551962ad 2024-11-15T22:14:35,134 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2a159d388ea2138ee8e2bed061a785ce, server=e611192d6313,43349,1731708869385 because future has completed 2024-11-15T22:14:35,134 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/51bf57bcf1afa9f625176363284d17f7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T22:14:35,135 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/13c3457b9645c9a8181ed9b4551962ad 2024-11-15T22:14:35,135 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 51bf57bcf1afa9f625176363284d17f7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66130599, jitterRate=-0.014577284455299377}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T22:14:35,136 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 51bf57bcf1afa9f625176363284d17f7 2024-11-15T22:14:35,136 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 51bf57bcf1afa9f625176363284d17f7: Running coprocessor pre-open hook at 1731708875101Writing region info on filesystem at 1731708875101Initializing all the Stores at 1731708875103 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708875104 (+1 ms)Cleaning up temporary data from old regions at 1731708875125 (+21 ms)Running coprocessor post-open hooks at 1731708875136 (+11 ms)Region opened successfully at 1731708875136 2024-11-15T22:14:35,136 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/13c3457b9645c9a8181ed9b4551962ad 2024-11-15T22:14:35,138 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for 13c3457b9645c9a8181ed9b4551962ad 2024-11-15T22:14:35,138 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for 13c3457b9645c9a8181ed9b4551962ad 2024-11-15T22:14:35,138 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ddd,1731708873681.51bf57bcf1afa9f625176363284d17f7., pid=37, masterSystemTime=1731708874824 2024-11-15T22:14:35,146 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for 13c3457b9645c9a8181ed9b4551962ad 2024-11-15T22:14:35,146 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=28 2024-11-15T22:14:35,146 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=28, state=SUCCESS, hasLock=false; OpenRegionProcedure a7fb991e3993a690fe7a0a1b064d5f5d, server=e611192d6313,41231,1731708869041 in 374 msec 2024-11-15T22:14:35,147 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ddd,1731708873681.51bf57bcf1afa9f625176363284d17f7. 2024-11-15T22:14:35,147 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened test,ddd,1731708873681.51bf57bcf1afa9f625176363284d17f7. 2024-11-15T22:14:35,147 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open test,ggg,1731708873681.1959cb95bb21ecdc677f7f5290173984. 2024-11-15T22:14:35,148 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => 1959cb95bb21ecdc677f7f5290173984, NAME => 'test,ggg,1731708873681.1959cb95bb21ecdc677f7f5290173984.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-15T22:14:35,148 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 1959cb95bb21ecdc677f7f5290173984 2024-11-15T22:14:35,148 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated test,ggg,1731708873681.1959cb95bb21ecdc677f7f5290173984.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:35,149 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for 1959cb95bb21ecdc677f7f5290173984 2024-11-15T22:14:35,149 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for 1959cb95bb21ecdc677f7f5290173984 2024-11-15T22:14:35,149 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=51bf57bcf1afa9f625176363284d17f7, regionState=OPEN, openSeqNum=2, regionLocation=e611192d6313,41231,1731708869041 2024-11-15T22:14:35,160 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 51bf57bcf1afa9f625176363284d17f7, server=e611192d6313,41231,1731708869041 because future has completed 2024-11-15T22:14:35,160 INFO [StoreOpener-1959cb95bb21ecdc677f7f5290173984-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 1959cb95bb21ecdc677f7f5290173984 2024-11-15T22:14:35,160 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/13c3457b9645c9a8181ed9b4551962ad/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T22:14:35,162 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened 13c3457b9645c9a8181ed9b4551962ad; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68211280, jitterRate=0.016427278518676758}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T22:14:35,163 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 13c3457b9645c9a8181ed9b4551962ad 2024-11-15T22:14:35,163 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for 13c3457b9645c9a8181ed9b4551962ad: Running coprocessor pre-open hook at 1731708875122Writing region info on filesystem at 1731708875122Initializing all the Stores at 1731708875126 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708875126Cleaning up temporary data from old regions at 1731708875139 (+13 ms)Running coprocessor post-open hooks at 1731708875163 (+24 ms)Region opened successfully at 1731708875163 2024-11-15T22:14:35,165 INFO [StoreOpener-1959cb95bb21ecdc677f7f5290173984-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1959cb95bb21ecdc677f7f5290173984 columnFamilyName family 2024-11-15T22:14:35,165 DEBUG [StoreOpener-1959cb95bb21ecdc677f7f5290173984-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:35,166 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1731708873681.13c3457b9645c9a8181ed9b4551962ad., pid=45, masterSystemTime=1731708874940 2024-11-15T22:14:35,166 INFO [StoreOpener-1959cb95bb21ecdc677f7f5290173984-1 {}] regionserver.HStore(327): Store=1959cb95bb21ecdc677f7f5290173984/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:14:35,166 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for 1959cb95bb21ecdc677f7f5290173984 2024-11-15T22:14:35,169 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/1959cb95bb21ecdc677f7f5290173984 2024-11-15T22:14:35,170 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/1959cb95bb21ecdc677f7f5290173984 2024-11-15T22:14:35,171 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for 1959cb95bb21ecdc677f7f5290173984 2024-11-15T22:14:35,172 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for 1959cb95bb21ecdc677f7f5290173984 2024-11-15T22:14:35,175 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39087 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=e611192d6313,41231,1731708869041, table=test, region=f64d8a749084ba25ea1b77db9a0d9f17. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T22:14:35,176 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1731708873681.13c3457b9645c9a8181ed9b4551962ad. 2024-11-15T22:14:35,176 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened test,hhh,1731708873681.13c3457b9645c9a8181ed9b4551962ad. 2024-11-15T22:14:35,176 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open test,,1731708873681.f2783381a48c13c2f75c07e5029903cb. 2024-11-15T22:14:35,176 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => f2783381a48c13c2f75c07e5029903cb, NAME => 'test,,1731708873681.f2783381a48c13c2f75c07e5029903cb.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-15T22:14:35,177 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f2783381a48c13c2f75c07e5029903cb 2024-11-15T22:14:35,177 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated test,,1731708873681.f2783381a48c13c2f75c07e5029903cb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:35,177 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for f2783381a48c13c2f75c07e5029903cb 2024-11-15T22:14:35,177 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for f2783381a48c13c2f75c07e5029903cb 2024-11-15T22:14:35,180 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for 1959cb95bb21ecdc677f7f5290173984 2024-11-15T22:14:35,181 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=8 2024-11-15T22:14:35,181 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 51bf57bcf1afa9f625176363284d17f7, server=e611192d6313,41231,1731708869041 in 466 msec 2024-11-15T22:14:35,181 INFO [StoreOpener-f2783381a48c13c2f75c07e5029903cb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f2783381a48c13c2f75c07e5029903cb 2024-11-15T22:14:35,186 INFO [StoreOpener-f2783381a48c13c2f75c07e5029903cb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f2783381a48c13c2f75c07e5029903cb columnFamilyName family 2024-11-15T22:14:35,186 DEBUG [StoreOpener-f2783381a48c13c2f75c07e5029903cb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:35,188 INFO [StoreOpener-f2783381a48c13c2f75c07e5029903cb-1 {}] regionserver.HStore(327): Store=f2783381a48c13c2f75c07e5029903cb/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:14:35,189 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for f2783381a48c13c2f75c07e5029903cb 2024-11-15T22:14:35,190 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=51bf57bcf1afa9f625176363284d17f7, ASSIGN in 761 msec 2024-11-15T22:14:35,192 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/f2783381a48c13c2f75c07e5029903cb 2024-11-15T22:14:35,193 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/f2783381a48c13c2f75c07e5029903cb 2024-11-15T22:14:35,193 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=13c3457b9645c9a8181ed9b4551962ad, regionState=OPEN, openSeqNum=2, regionLocation=e611192d6313,43349,1731708869385 2024-11-15T22:14:35,194 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for f2783381a48c13c2f75c07e5029903cb 2024-11-15T22:14:35,194 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for f2783381a48c13c2f75c07e5029903cb 2024-11-15T22:14:35,197 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/1959cb95bb21ecdc677f7f5290173984/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T22:14:35,198 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for f2783381a48c13c2f75c07e5029903cb 2024-11-15T22:14:35,198 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened 1959cb95bb21ecdc677f7f5290173984; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66969378, jitterRate=-0.002078503370285034}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T22:14:35,198 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1959cb95bb21ecdc677f7f5290173984 2024-11-15T22:14:35,199 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for 1959cb95bb21ecdc677f7f5290173984: Running coprocessor pre-open hook at 1731708875149Writing region info on filesystem at 1731708875149Initializing all the Stores at 1731708875152 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708875152Cleaning up temporary data from old regions at 1731708875172 (+20 ms)Running coprocessor post-open hooks at 1731708875198 (+26 ms)Region opened successfully at 1731708875199 (+1 ms) 2024-11-15T22:14:35,201 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ggg,1731708873681.1959cb95bb21ecdc677f7f5290173984., pid=35, masterSystemTime=1731708874824 2024-11-15T22:14:35,202 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,43349,1731708869385 because future has completed 2024-11-15T22:14:35,207 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a7fb991e3993a690fe7a0a1b064d5f5d, ASSIGN in 778 msec 2024-11-15T22:14:35,208 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ggg,1731708873681.1959cb95bb21ecdc677f7f5290173984. 2024-11-15T22:14:35,208 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened test,ggg,1731708873681.1959cb95bb21ecdc677f7f5290173984. 2024-11-15T22:14:35,208 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open test,uuu,1731708873681.e87a82b283b7251342a664ba3876c98e. 2024-11-15T22:14:35,209 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => e87a82b283b7251342a664ba3876c98e, NAME => 'test,uuu,1731708873681.e87a82b283b7251342a664ba3876c98e.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-15T22:14:35,210 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39087 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=e611192d6313,43349,1731708869385, table=test, region=93deaad3e3229244196e8daa9478371a. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T22:14:35,210 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39087 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=e611192d6313,43349,1731708869385, table=test, region=2a159d388ea2138ee8e2bed061a785ce. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T22:14:35,211 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test e87a82b283b7251342a664ba3876c98e 2024-11-15T22:14:35,211 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated test,uuu,1731708873681.e87a82b283b7251342a664ba3876c98e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:35,211 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for e87a82b283b7251342a664ba3876c98e 2024-11-15T22:14:35,211 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for e87a82b283b7251342a664ba3876c98e 2024-11-15T22:14:35,216 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/f2783381a48c13c2f75c07e5029903cb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T22:14:35,218 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=12 2024-11-15T22:14:35,219 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,43349,1731708869385 in 372 msec 2024-11-15T22:14:35,220 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened f2783381a48c13c2f75c07e5029903cb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61251107, jitterRate=-0.08728738129138947}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T22:14:35,220 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f2783381a48c13c2f75c07e5029903cb 2024-11-15T22:14:35,220 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for f2783381a48c13c2f75c07e5029903cb: Running coprocessor pre-open hook at 1731708875177Writing region info on filesystem at 1731708875177Initializing all the Stores at 1731708875180 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708875180Cleaning up temporary data from old regions at 1731708875194 (+14 ms)Running coprocessor post-open hooks at 1731708875220 (+26 ms)Region opened successfully at 1731708875220 2024-11-15T22:14:35,224 INFO [StoreOpener-e87a82b283b7251342a664ba3876c98e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region e87a82b283b7251342a664ba3876c98e 2024-11-15T22:14:35,225 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1731708873681.f2783381a48c13c2f75c07e5029903cb., pid=43, masterSystemTime=1731708874940 2024-11-15T22:14:35,229 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for test,,1731708873681.f2783381a48c13c2f75c07e5029903cb. 2024-11-15T22:14:35,229 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened test,,1731708873681.f2783381a48c13c2f75c07e5029903cb. 2024-11-15T22:14:35,230 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(132): Open test,bbb,1731708873681.897a7bb57a002d365489ce583eb7f84f. 2024-11-15T22:14:35,230 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7752): Opening region: {ENCODED => 897a7bb57a002d365489ce583eb7f84f, NAME => 'test,bbb,1731708873681.897a7bb57a002d365489ce583eb7f84f.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-11-15T22:14:35,230 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 897a7bb57a002d365489ce583eb7f84f 2024-11-15T22:14:35,231 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(898): Instantiated test,bbb,1731708873681.897a7bb57a002d365489ce583eb7f84f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:35,231 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7794): checking encryption for 897a7bb57a002d365489ce583eb7f84f 2024-11-15T22:14:35,231 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7797): checking classloading for 897a7bb57a002d365489ce583eb7f84f 2024-11-15T22:14:35,233 INFO [StoreOpener-e87a82b283b7251342a664ba3876c98e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e87a82b283b7251342a664ba3876c98e columnFamilyName family 2024-11-15T22:14:35,234 DEBUG [StoreOpener-e87a82b283b7251342a664ba3876c98e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:35,235 INFO [StoreOpener-e87a82b283b7251342a664ba3876c98e-1 {}] regionserver.HStore(327): Store=e87a82b283b7251342a664ba3876c98e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:14:35,235 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for e87a82b283b7251342a664ba3876c98e 2024-11-15T22:14:35,237 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/e87a82b283b7251342a664ba3876c98e 2024-11-15T22:14:35,240 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/e87a82b283b7251342a664ba3876c98e 2024-11-15T22:14:35,241 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for e87a82b283b7251342a664ba3876c98e 2024-11-15T22:14:35,242 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for e87a82b283b7251342a664ba3876c98e 2024-11-15T22:14:35,245 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN in 799 msec 2024-11-15T22:14:35,246 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for e87a82b283b7251342a664ba3876c98e 2024-11-15T22:14:35,247 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=f2783381a48c13c2f75c07e5029903cb, regionState=OPEN, openSeqNum=2, regionLocation=e611192d6313,43349,1731708869385 2024-11-15T22:14:35,248 INFO [StoreOpener-897a7bb57a002d365489ce583eb7f84f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 897a7bb57a002d365489ce583eb7f84f 2024-11-15T22:14:35,255 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure f2783381a48c13c2f75c07e5029903cb, server=e611192d6313,43349,1731708869385 because future has completed 2024-11-15T22:14:35,262 INFO [StoreOpener-897a7bb57a002d365489ce583eb7f84f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 897a7bb57a002d365489ce583eb7f84f columnFamilyName family 2024-11-15T22:14:35,262 DEBUG [StoreOpener-897a7bb57a002d365489ce583eb7f84f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:35,268 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/e87a82b283b7251342a664ba3876c98e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T22:14:35,268 INFO [StoreOpener-897a7bb57a002d365489ce583eb7f84f-1 {}] regionserver.HStore(327): Store=897a7bb57a002d365489ce583eb7f84f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:14:35,268 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1038): replaying wal for 897a7bb57a002d365489ce583eb7f84f 2024-11-15T22:14:35,273 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=1959cb95bb21ecdc677f7f5290173984, regionState=OPEN, openSeqNum=2, regionLocation=e611192d6313,41231,1731708869041 2024-11-15T22:14:35,273 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened e87a82b283b7251342a664ba3876c98e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71429291, jitterRate=0.06437937915325165}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T22:14:35,273 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e87a82b283b7251342a664ba3876c98e 2024-11-15T22:14:35,273 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for e87a82b283b7251342a664ba3876c98e: Running coprocessor pre-open hook at 1731708875211Writing region info on filesystem at 1731708875211Initializing all the Stores at 1731708875215 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708875215Cleaning up temporary data from old regions at 1731708875242 (+27 ms)Running coprocessor post-open hooks at 1731708875273 (+31 ms)Region opened successfully at 1731708875273 2024-11-15T22:14:35,276 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/897a7bb57a002d365489ce583eb7f84f 2024-11-15T22:14:35,278 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1959cb95bb21ecdc677f7f5290173984, server=e611192d6313,41231,1731708869041 because future has completed 2024-11-15T22:14:35,280 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1731708873681.e87a82b283b7251342a664ba3876c98e., pid=34, masterSystemTime=1731708874824 2024-11-15T22:14:35,283 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/897a7bb57a002d365489ce583eb7f84f 2024-11-15T22:14:35,288 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1048): stopping wal replay for 897a7bb57a002d365489ce583eb7f84f 2024-11-15T22:14:35,288 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1060): Cleaning up temporary data for 897a7bb57a002d365489ce583eb7f84f 2024-11-15T22:14:35,290 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39087 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=e611192d6313,41231,1731708869041, table=test, region=1959cb95bb21ecdc677f7f5290173984. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T22:14:35,290 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39087 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=e611192d6313,41231,1731708869041, table=test, region=f64d8a749084ba25ea1b77db9a0d9f17. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T22:14:35,294 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for test,uuu,1731708873681.e87a82b283b7251342a664ba3876c98e. 2024-11-15T22:14:35,294 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened test,uuu,1731708873681.e87a82b283b7251342a664ba3876c98e. 2024-11-15T22:14:35,295 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open test,jjj,1731708873681.36b8b3bed3adb94f5901f1f3809b64fa. 2024-11-15T22:14:35,295 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => 36b8b3bed3adb94f5901f1f3809b64fa, NAME => 'test,jjj,1731708873681.36b8b3bed3adb94f5901f1f3809b64fa.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-15T22:14:35,295 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 36b8b3bed3adb94f5901f1f3809b64fa 2024-11-15T22:14:35,295 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated test,jjj,1731708873681.36b8b3bed3adb94f5901f1f3809b64fa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:35,295 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for 36b8b3bed3adb94f5901f1f3809b64fa 2024-11-15T22:14:35,295 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for 36b8b3bed3adb94f5901f1f3809b64fa 2024-11-15T22:14:35,305 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1093): writing seq id for 897a7bb57a002d365489ce583eb7f84f 2024-11-15T22:14:35,307 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=5 2024-11-15T22:14:35,308 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure f2783381a48c13c2f75c07e5029903cb, server=e611192d6313,43349,1731708869385 in 514 msec 2024-11-15T22:14:35,308 INFO [StoreOpener-36b8b3bed3adb94f5901f1f3809b64fa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 36b8b3bed3adb94f5901f1f3809b64fa 2024-11-15T22:14:35,312 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f2783381a48c13c2f75c07e5029903cb, ASSIGN in 888 msec 2024-11-15T22:14:35,315 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=e87a82b283b7251342a664ba3876c98e, regionState=OPEN, openSeqNum=2, regionLocation=e611192d6313,41231,1731708869041 2024-11-15T22:14:35,316 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39087 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=e611192d6313,43349,1731708869385, table=test, region=93deaad3e3229244196e8daa9478371a. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T22:14:35,316 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39087 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=e611192d6313,43349,1731708869385, table=test, region=2a159d388ea2138ee8e2bed061a785ce. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T22:14:35,320 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure e87a82b283b7251342a664ba3876c98e, server=e611192d6313,41231,1731708869041 because future has completed 2024-11-15T22:14:35,327 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=11 2024-11-15T22:14:35,327 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 1959cb95bb21ecdc677f7f5290173984, server=e611192d6313,41231,1731708869041 in 646 msec 2024-11-15T22:14:35,330 INFO [StoreOpener-36b8b3bed3adb94f5901f1f3809b64fa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 36b8b3bed3adb94f5901f1f3809b64fa columnFamilyName family 2024-11-15T22:14:35,330 DEBUG [StoreOpener-36b8b3bed3adb94f5901f1f3809b64fa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:35,332 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=1959cb95bb21ecdc677f7f5290173984, ASSIGN in 907 msec 2024-11-15T22:14:35,333 INFO [StoreOpener-36b8b3bed3adb94f5901f1f3809b64fa-1 {}] regionserver.HStore(327): Store=36b8b3bed3adb94f5901f1f3809b64fa/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:14:35,333 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for 36b8b3bed3adb94f5901f1f3809b64fa 2024-11-15T22:14:35,337 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/36b8b3bed3adb94f5901f1f3809b64fa 2024-11-15T22:14:35,344 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=25 2024-11-15T22:14:35,344 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure e87a82b283b7251342a664ba3876c98e, server=e611192d6313,41231,1731708869041 in 679 msec 2024-11-15T22:14:35,345 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/897a7bb57a002d365489ce583eb7f84f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T22:14:35,348 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/36b8b3bed3adb94f5901f1f3809b64fa 2024-11-15T22:14:35,349 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1114): Opened 897a7bb57a002d365489ce583eb7f84f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59565466, jitterRate=-0.11240538954734802}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T22:14:35,349 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 897a7bb57a002d365489ce583eb7f84f 2024-11-15T22:14:35,349 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1006): Region open journal for 897a7bb57a002d365489ce583eb7f84f: Running coprocessor pre-open hook at 1731708875231Writing region info on filesystem at 1731708875231Initializing all the Stores at 1731708875237 (+6 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708875237Cleaning up temporary data from old regions at 1731708875288 (+51 ms)Running coprocessor post-open hooks at 1731708875349 (+61 ms)Region opened successfully at 1731708875349 2024-11-15T22:14:35,349 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e87a82b283b7251342a664ba3876c98e, ASSIGN in 925 msec 2024-11-15T22:14:35,350 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for 36b8b3bed3adb94f5901f1f3809b64fa 2024-11-15T22:14:35,350 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for 36b8b3bed3adb94f5901f1f3809b64fa 2024-11-15T22:14:35,354 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for 36b8b3bed3adb94f5901f1f3809b64fa 2024-11-15T22:14:35,356 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2236): Post open deploy tasks for test,bbb,1731708873681.897a7bb57a002d365489ce583eb7f84f., pid=44, masterSystemTime=1731708874940 2024-11-15T22:14:35,364 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2266): Finished post open deploy task for test,bbb,1731708873681.897a7bb57a002d365489ce583eb7f84f. 2024-11-15T22:14:35,364 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(153): Opened test,bbb,1731708873681.897a7bb57a002d365489ce583eb7f84f. 2024-11-15T22:14:35,364 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open test,nnn,1731708873681.c56d152c07a95a189ac65341ff4db1fa. 2024-11-15T22:14:35,365 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => c56d152c07a95a189ac65341ff4db1fa, NAME => 'test,nnn,1731708873681.c56d152c07a95a189ac65341ff4db1fa.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-15T22:14:35,365 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c56d152c07a95a189ac65341ff4db1fa 2024-11-15T22:14:35,365 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated test,nnn,1731708873681.c56d152c07a95a189ac65341ff4db1fa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:35,365 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for c56d152c07a95a189ac65341ff4db1fa 2024-11-15T22:14:35,365 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for c56d152c07a95a189ac65341ff4db1fa 2024-11-15T22:14:35,368 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/36b8b3bed3adb94f5901f1f3809b64fa/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T22:14:35,368 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=15 2024-11-15T22:14:35,368 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure 2a159d388ea2138ee8e2bed061a785ce, server=e611192d6313,43349,1731708869385 in 478 msec 2024-11-15T22:14:35,369 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened 36b8b3bed3adb94f5901f1f3809b64fa; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67385725, jitterRate=0.004125550389289856}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T22:14:35,369 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 36b8b3bed3adb94f5901f1f3809b64fa 2024-11-15T22:14:35,369 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for 36b8b3bed3adb94f5901f1f3809b64fa: Running coprocessor pre-open hook at 1731708875296Writing region info on filesystem at 1731708875296Initializing all the Stores at 1731708875301 (+5 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708875301Cleaning up temporary data from old regions at 1731708875350 (+49 ms)Running coprocessor post-open hooks at 1731708875369 (+19 ms)Region opened successfully at 1731708875369 2024-11-15T22:14:35,371 INFO [StoreOpener-c56d152c07a95a189ac65341ff4db1fa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c56d152c07a95a189ac65341ff4db1fa 2024-11-15T22:14:35,375 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN in 949 msec 2024-11-15T22:14:35,378 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=897a7bb57a002d365489ce583eb7f84f, regionState=OPEN, openSeqNum=2, regionLocation=e611192d6313,43349,1731708869385 2024-11-15T22:14:35,380 INFO [StoreOpener-c56d152c07a95a189ac65341ff4db1fa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c56d152c07a95a189ac65341ff4db1fa columnFamilyName family 2024-11-15T22:14:35,381 DEBUG [StoreOpener-c56d152c07a95a189ac65341ff4db1fa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:35,382 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for test,jjj,1731708873681.36b8b3bed3adb94f5901f1f3809b64fa., pid=38, masterSystemTime=1731708874824 2024-11-15T22:14:35,387 INFO [StoreOpener-c56d152c07a95a189ac65341ff4db1fa-1 {}] regionserver.HStore(327): Store=c56d152c07a95a189ac65341ff4db1fa/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:14:35,388 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for c56d152c07a95a189ac65341ff4db1fa 2024-11-15T22:14:35,392 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=6, state=RUNNABLE, hasLock=true; OpenRegionProcedure 897a7bb57a002d365489ce583eb7f84f, server=e611192d6313,43349,1731708869385 because future has completed 2024-11-15T22:14:35,392 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for test,jjj,1731708873681.36b8b3bed3adb94f5901f1f3809b64fa. 2024-11-15T22:14:35,392 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/c56d152c07a95a189ac65341ff4db1fa 2024-11-15T22:14:35,392 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened test,jjj,1731708873681.36b8b3bed3adb94f5901f1f3809b64fa. 2024-11-15T22:14:35,392 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open test,ppp,1731708873681.83048e0be415b190a19f64bc2e49d3fe. 2024-11-15T22:14:35,393 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => 83048e0be415b190a19f64bc2e49d3fe, NAME => 'test,ppp,1731708873681.83048e0be415b190a19f64bc2e49d3fe.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-15T22:14:35,393 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 83048e0be415b190a19f64bc2e49d3fe 2024-11-15T22:14:35,393 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated test,ppp,1731708873681.83048e0be415b190a19f64bc2e49d3fe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:35,394 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for 83048e0be415b190a19f64bc2e49d3fe 2024-11-15T22:14:35,394 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for 83048e0be415b190a19f64bc2e49d3fe 2024-11-15T22:14:35,396 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/c56d152c07a95a189ac65341ff4db1fa 2024-11-15T22:14:35,400 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for c56d152c07a95a189ac65341ff4db1fa 2024-11-15T22:14:35,400 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for c56d152c07a95a189ac65341ff4db1fa 2024-11-15T22:14:35,401 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39087 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=e611192d6313,41231,1731708869041, table=test, region=f64d8a749084ba25ea1b77db9a0d9f17. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T22:14:35,402 INFO [StoreOpener-83048e0be415b190a19f64bc2e49d3fe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 83048e0be415b190a19f64bc2e49d3fe 2024-11-15T22:14:35,409 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for c56d152c07a95a189ac65341ff4db1fa 2024-11-15T22:14:35,410 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=6 2024-11-15T22:14:35,411 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure 897a7bb57a002d365489ce583eb7f84f, server=e611192d6313,43349,1731708869385 in 606 msec 2024-11-15T22:14:35,413 INFO [StoreOpener-83048e0be415b190a19f64bc2e49d3fe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 83048e0be415b190a19f64bc2e49d3fe columnFamilyName family 2024-11-15T22:14:35,413 DEBUG [StoreOpener-83048e0be415b190a19f64bc2e49d3fe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:35,415 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=897a7bb57a002d365489ce583eb7f84f, ASSIGN in 991 msec 2024-11-15T22:14:35,416 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/c56d152c07a95a189ac65341ff4db1fa/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T22:14:35,418 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=36b8b3bed3adb94f5901f1f3809b64fa, regionState=OPEN, openSeqNum=2, regionLocation=e611192d6313,41231,1731708869041 2024-11-15T22:14:35,420 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened c56d152c07a95a189ac65341ff4db1fa; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62216346, jitterRate=-0.07290419936180115}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T22:14:35,420 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c56d152c07a95a189ac65341ff4db1fa 2024-11-15T22:14:35,420 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for c56d152c07a95a189ac65341ff4db1fa: Running coprocessor pre-open hook at 1731708875366Writing region info on filesystem at 1731708875366Initializing all the Stores at 1731708875368 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708875368Cleaning up temporary data from old regions at 1731708875400 (+32 ms)Running coprocessor post-open hooks at 1731708875420 (+20 ms)Region opened successfully at 1731708875420 2024-11-15T22:14:35,420 INFO [StoreOpener-83048e0be415b190a19f64bc2e49d3fe-1 {}] regionserver.HStore(327): Store=83048e0be415b190a19f64bc2e49d3fe/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:14:35,420 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for 83048e0be415b190a19f64bc2e49d3fe 2024-11-15T22:14:35,422 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 36b8b3bed3adb94f5901f1f3809b64fa, server=e611192d6313,41231,1731708869041 because future has completed 2024-11-15T22:14:35,424 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39087 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=e611192d6313,43349,1731708869385, table=test, region=93deaad3e3229244196e8daa9478371a. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T22:14:35,424 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/83048e0be415b190a19f64bc2e49d3fe 2024-11-15T22:14:35,428 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for test,nnn,1731708873681.c56d152c07a95a189ac65341ff4db1fa., pid=47, masterSystemTime=1731708874940 2024-11-15T22:14:35,428 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/83048e0be415b190a19f64bc2e49d3fe 2024-11-15T22:14:35,431 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=9 2024-11-15T22:14:35,431 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure 93deaad3e3229244196e8daa9478371a, server=e611192d6313,43349,1731708869385 in 784 msec 2024-11-15T22:14:35,433 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for 83048e0be415b190a19f64bc2e49d3fe 2024-11-15T22:14:35,433 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for 83048e0be415b190a19f64bc2e49d3fe 2024-11-15T22:14:35,436 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=93deaad3e3229244196e8daa9478371a, ASSIGN in 1.0110 sec 2024-11-15T22:14:35,436 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for test,nnn,1731708873681.c56d152c07a95a189ac65341ff4db1fa. 2024-11-15T22:14:35,436 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened test,nnn,1731708873681.c56d152c07a95a189ac65341ff4db1fa. 2024-11-15T22:14:35,438 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=c56d152c07a95a189ac65341ff4db1fa, regionState=OPEN, openSeqNum=2, regionLocation=e611192d6313,43349,1731708869385 2024-11-15T22:14:35,442 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure c56d152c07a95a189ac65341ff4db1fa, server=e611192d6313,43349,1731708869385 because future has completed 2024-11-15T22:14:35,448 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for 83048e0be415b190a19f64bc2e49d3fe 2024-11-15T22:14:35,453 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=14 2024-11-15T22:14:35,453 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 36b8b3bed3adb94f5901f1f3809b64fa, server=e611192d6313,41231,1731708869041 in 736 msec 2024-11-15T22:14:35,453 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/83048e0be415b190a19f64bc2e49d3fe/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T22:14:35,454 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened 83048e0be415b190a19f64bc2e49d3fe; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60049933, jitterRate=-0.10518626868724823}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T22:14:35,454 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 83048e0be415b190a19f64bc2e49d3fe 2024-11-15T22:14:35,454 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for 83048e0be415b190a19f64bc2e49d3fe: Running coprocessor pre-open hook at 1731708875394Writing region info on filesystem at 1731708875394Initializing all the Stores at 1731708875400 (+6 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708875400Cleaning up temporary data from old regions at 1731708875433 (+33 ms)Running coprocessor post-open hooks at 1731708875454 (+21 ms)Region opened successfully at 1731708875454 2024-11-15T22:14:35,456 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1731708873681.83048e0be415b190a19f64bc2e49d3fe., pid=40, masterSystemTime=1731708874824 2024-11-15T22:14:35,461 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ppp,1731708873681.83048e0be415b190a19f64bc2e49d3fe. 2024-11-15T22:14:35,461 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened test,ppp,1731708873681.83048e0be415b190a19f64bc2e49d3fe. 2024-11-15T22:14:35,462 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=36b8b3bed3adb94f5901f1f3809b64fa, ASSIGN in 1.0330 sec 2024-11-15T22:14:35,464 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=83048e0be415b190a19f64bc2e49d3fe, regionState=OPEN, openSeqNum=2, regionLocation=e611192d6313,41231,1731708869041 2024-11-15T22:14:35,468 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 83048e0be415b190a19f64bc2e49d3fe, server=e611192d6313,41231,1731708869041 because future has completed 2024-11-15T22:14:35,475 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=18 2024-11-15T22:14:35,475 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure c56d152c07a95a189ac65341ff4db1fa, server=e611192d6313,43349,1731708869385 in 569 msec 2024-11-15T22:14:35,479 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=c56d152c07a95a189ac65341ff4db1fa, ASSIGN in 1.0550 sec 2024-11-15T22:14:35,487 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=20 2024-11-15T22:14:35,487 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 83048e0be415b190a19f64bc2e49d3fe, server=e611192d6313,41231,1731708869041 in 757 msec 2024-11-15T22:14:35,490 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=83048e0be415b190a19f64bc2e49d3fe, ASSIGN in 1.0670 sec 2024-11-15T22:14:35,497 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=17 2024-11-15T22:14:35,497 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure f64d8a749084ba25ea1b77db9a0d9f17, server=e611192d6313,41231,1731708869041 in 775 msec 2024-11-15T22:14:35,501 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f64d8a749084ba25ea1b77db9a0d9f17, ASSIGN in 1.0770 sec 2024-11-15T22:14:35,503 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=4381d14830a288bee3aeb42195dd8768, regionState=OPEN, openSeqNum=2, regionLocation=e611192d6313,41231,1731708869041 2024-11-15T22:14:35,506 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39087 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=e611192d6313,41231,1731708869041, table=test, region=4381d14830a288bee3aeb42195dd8768. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T22:14:35,508 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4381d14830a288bee3aeb42195dd8768, server=e611192d6313,41231,1731708869041 because future has completed 2024-11-15T22:14:35,508 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 819421a6406c0ebb5635c399b2c0303d, server=e611192d6313,43349,1731708869385}] 2024-11-15T22:14:35,521 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=24 2024-11-15T22:14:35,522 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure 4381d14830a288bee3aeb42195dd8768, server=e611192d6313,41231,1731708869041 in 770 msec 2024-11-15T22:14:35,525 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=4381d14830a288bee3aeb42195dd8768, ASSIGN in 1.1020 sec 2024-11-15T22:14:35,526 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=c1cecce90107a4c37b7d91651975993f, regionState=OPENING, regionLocation=e611192d6313,38729,1731708868111 2024-11-15T22:14:35,529 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=f9e6155d6c82d9d9a5c9449be6d7ce5f, regionState=OPENING, regionLocation=e611192d6313,38729,1731708868111 2024-11-15T22:14:35,530 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=7502fc847b4b97e1ace9a22ec1c068e0, regionState=OPENING, regionLocation=e611192d6313,38729,1731708868111 2024-11-15T22:14:35,531 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=a602f0e8bd42c685584b8b985174ddea, regionState=OPENING, regionLocation=e611192d6313,38729,1731708868111 2024-11-15T22:14:35,532 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=de9977b187fcaabda585b015a568e905, regionState=OPENING, regionLocation=e611192d6313,38729,1731708868111 2024-11-15T22:14:35,533 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c1cecce90107a4c37b7d91651975993f, ASSIGN because future has completed 2024-11-15T22:14:35,534 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure c1cecce90107a4c37b7d91651975993f, server=e611192d6313,38729,1731708868111}] 2024-11-15T22:14:35,537 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f9e6155d6c82d9d9a5c9449be6d7ce5f, ASSIGN because future has completed 2024-11-15T22:14:35,539 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=7502fc847b4b97e1ace9a22ec1c068e0, ASSIGN because future has completed 2024-11-15T22:14:35,541 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a602f0e8bd42c685584b8b985174ddea, ASSIGN because future has completed 2024-11-15T22:14:35,543 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=de9977b187fcaabda585b015a568e905, ASSIGN because future has completed 2024-11-15T22:14:35,548 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure de9977b187fcaabda585b015a568e905, server=e611192d6313,38729,1731708868111}] 2024-11-15T22:14:35,560 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure a602f0e8bd42c685584b8b985174ddea, server=e611192d6313,38729,1731708868111}] 2024-11-15T22:14:35,570 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7502fc847b4b97e1ace9a22ec1c068e0, server=e611192d6313,38729,1731708868111}] 2024-11-15T22:14:35,579 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure f9e6155d6c82d9d9a5c9449be6d7ce5f, server=e611192d6313,38729,1731708868111}] 2024-11-15T22:14:35,591 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=1732b30f3d981289760d152f788aff6f, regionState=OPENING, regionLocation=e611192d6313,38729,1731708868111 2024-11-15T22:14:35,593 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=ecaeddccb9b2987f43bb050f0a105976, regionState=OPENING, regionLocation=e611192d6313,38729,1731708868111 2024-11-15T22:14:35,594 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=2424ff4b6d907aafacef6f661d88ef95, regionState=OPENING, regionLocation=e611192d6313,38729,1731708868111 2024-11-15T22:14:35,598 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=1732b30f3d981289760d152f788aff6f, ASSIGN because future has completed 2024-11-15T22:14:35,598 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1732b30f3d981289760d152f788aff6f, server=e611192d6313,38729,1731708868111}] 2024-11-15T22:14:35,606 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ecaeddccb9b2987f43bb050f0a105976, ASSIGN because future has completed 2024-11-15T22:14:35,607 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=2424ff4b6d907aafacef6f661d88ef95, ASSIGN because future has completed 2024-11-15T22:14:35,614 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2424ff4b6d907aafacef6f661d88ef95, server=e611192d6313,38729,1731708868111}] 2024-11-15T22:14:35,622 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure ecaeddccb9b2987f43bb050f0a105976, server=e611192d6313,38729,1731708868111}] 2024-11-15T22:14:35,670 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(132): Open test,sss,1731708873681.819421a6406c0ebb5635c399b2c0303d. 2024-11-15T22:14:35,670 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7752): Opening region: {ENCODED => 819421a6406c0ebb5635c399b2c0303d, NAME => 'test,sss,1731708873681.819421a6406c0ebb5635c399b2c0303d.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-15T22:14:35,670 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 819421a6406c0ebb5635c399b2c0303d 2024-11-15T22:14:35,671 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(898): Instantiated test,sss,1731708873681.819421a6406c0ebb5635c399b2c0303d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:35,671 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7794): checking encryption for 819421a6406c0ebb5635c399b2c0303d 2024-11-15T22:14:35,671 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7797): checking classloading for 819421a6406c0ebb5635c399b2c0303d 2024-11-15T22:14:35,677 INFO [StoreOpener-819421a6406c0ebb5635c399b2c0303d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 819421a6406c0ebb5635c399b2c0303d 2024-11-15T22:14:35,682 INFO [StoreOpener-819421a6406c0ebb5635c399b2c0303d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 819421a6406c0ebb5635c399b2c0303d columnFamilyName family 2024-11-15T22:14:35,682 DEBUG [StoreOpener-819421a6406c0ebb5635c399b2c0303d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:35,683 INFO [StoreOpener-819421a6406c0ebb5635c399b2c0303d-1 {}] regionserver.HStore(327): Store=819421a6406c0ebb5635c399b2c0303d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:14:35,684 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1038): replaying wal for 819421a6406c0ebb5635c399b2c0303d 2024-11-15T22:14:35,688 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/819421a6406c0ebb5635c399b2c0303d 2024-11-15T22:14:35,689 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/819421a6406c0ebb5635c399b2c0303d 2024-11-15T22:14:35,689 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1048): stopping wal replay for 819421a6406c0ebb5635c399b2c0303d 2024-11-15T22:14:35,689 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1060): Cleaning up temporary data for 819421a6406c0ebb5635c399b2c0303d 2024-11-15T22:14:35,694 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1093): writing seq id for 819421a6406c0ebb5635c399b2c0303d 2024-11-15T22:14:35,697 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(132): Open test,fff,1731708873681.ecaeddccb9b2987f43bb050f0a105976. 2024-11-15T22:14:35,697 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7752): Opening region: {ENCODED => ecaeddccb9b2987f43bb050f0a105976, NAME => 'test,fff,1731708873681.ecaeddccb9b2987f43bb050f0a105976.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-15T22:14:35,698 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ecaeddccb9b2987f43bb050f0a105976 2024-11-15T22:14:35,698 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(898): Instantiated test,fff,1731708873681.ecaeddccb9b2987f43bb050f0a105976.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:35,698 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7794): checking encryption for ecaeddccb9b2987f43bb050f0a105976 2024-11-15T22:14:35,698 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7797): checking classloading for ecaeddccb9b2987f43bb050f0a105976 2024-11-15T22:14:35,713 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/819421a6406c0ebb5635c399b2c0303d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T22:14:35,715 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1114): Opened 819421a6406c0ebb5635c399b2c0303d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63357607, jitterRate=-0.05589808523654938}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T22:14:35,715 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 819421a6406c0ebb5635c399b2c0303d 2024-11-15T22:14:35,716 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1006): Region open journal for 819421a6406c0ebb5635c399b2c0303d: Running coprocessor pre-open hook at 1731708875671Writing region info on filesystem at 1731708875671Initializing all the Stores at 1731708875675 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708875675Cleaning up temporary data from old regions at 1731708875689 (+14 ms)Running coprocessor post-open hooks at 1731708875715 (+26 ms)Region opened successfully at 1731708875715 2024-11-15T22:14:35,717 INFO [StoreOpener-ecaeddccb9b2987f43bb050f0a105976-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ecaeddccb9b2987f43bb050f0a105976 2024-11-15T22:14:35,718 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1731708873681.819421a6406c0ebb5635c399b2c0303d., pid=48, masterSystemTime=1731708875662 2024-11-15T22:14:35,720 INFO [StoreOpener-ecaeddccb9b2987f43bb050f0a105976-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ecaeddccb9b2987f43bb050f0a105976 columnFamilyName family 2024-11-15T22:14:35,720 DEBUG [StoreOpener-ecaeddccb9b2987f43bb050f0a105976-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:35,722 INFO [StoreOpener-ecaeddccb9b2987f43bb050f0a105976-1 {}] regionserver.HStore(327): Store=ecaeddccb9b2987f43bb050f0a105976/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:14:35,722 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1038): replaying wal for ecaeddccb9b2987f43bb050f0a105976 2024-11-15T22:14:35,724 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/ecaeddccb9b2987f43bb050f0a105976 2024-11-15T22:14:35,724 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/ecaeddccb9b2987f43bb050f0a105976 2024-11-15T22:14:35,725 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1048): stopping wal replay for ecaeddccb9b2987f43bb050f0a105976 2024-11-15T22:14:35,725 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1060): Cleaning up temporary data for ecaeddccb9b2987f43bb050f0a105976 2024-11-15T22:14:35,726 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1731708873681.819421a6406c0ebb5635c399b2c0303d. 2024-11-15T22:14:35,726 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(153): Opened test,sss,1731708873681.819421a6406c0ebb5635c399b2c0303d. 2024-11-15T22:14:35,729 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=819421a6406c0ebb5635c399b2c0303d, regionState=OPEN, openSeqNum=2, regionLocation=e611192d6313,43349,1731708869385 2024-11-15T22:14:35,729 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1093): writing seq id for ecaeddccb9b2987f43bb050f0a105976 2024-11-15T22:14:35,733 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=48, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 819421a6406c0ebb5635c399b2c0303d, server=e611192d6313,43349,1731708869385 because future has completed 2024-11-15T22:14:35,734 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/ecaeddccb9b2987f43bb050f0a105976/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T22:14:35,735 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1114): Opened ecaeddccb9b2987f43bb050f0a105976; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72319005, jitterRate=0.07763715088367462}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T22:14:35,735 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ecaeddccb9b2987f43bb050f0a105976 2024-11-15T22:14:35,736 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1006): Region open journal for ecaeddccb9b2987f43bb050f0a105976: Running coprocessor pre-open hook at 1731708875698Writing region info on filesystem at 1731708875698Initializing all the Stores at 1731708875700 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708875700Cleaning up temporary data from old regions at 1731708875725 (+25 ms)Running coprocessor post-open hooks at 1731708875736 (+11 ms)Region opened successfully at 1731708875736 2024-11-15T22:14:35,741 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1731708873681.ecaeddccb9b2987f43bb050f0a105976., pid=56, masterSystemTime=1731708875689 2024-11-15T22:14:35,743 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=23 2024-11-15T22:14:35,743 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 819421a6406c0ebb5635c399b2c0303d, server=e611192d6313,43349,1731708869385 in 227 msec 2024-11-15T22:14:35,746 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1731708873681.ecaeddccb9b2987f43bb050f0a105976. 2024-11-15T22:14:35,746 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(153): Opened test,fff,1731708873681.ecaeddccb9b2987f43bb050f0a105976. 2024-11-15T22:14:35,747 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(132): Open test,lll,1731708873681.de9977b187fcaabda585b015a568e905. 2024-11-15T22:14:35,747 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7752): Opening region: {ENCODED => de9977b187fcaabda585b015a568e905, NAME => 'test,lll,1731708873681.de9977b187fcaabda585b015a568e905.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-15T22:14:35,747 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test de9977b187fcaabda585b015a568e905 2024-11-15T22:14:35,747 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(898): Instantiated test,lll,1731708873681.de9977b187fcaabda585b015a568e905.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:35,747 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7794): checking encryption for de9977b187fcaabda585b015a568e905 2024-11-15T22:14:35,747 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7797): checking classloading for de9977b187fcaabda585b015a568e905 2024-11-15T22:14:35,749 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN in 1.3240 sec 2024-11-15T22:14:35,751 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=ecaeddccb9b2987f43bb050f0a105976, regionState=OPEN, openSeqNum=2, regionLocation=e611192d6313,38729,1731708868111 2024-11-15T22:14:35,752 INFO [StoreOpener-de9977b187fcaabda585b015a568e905-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region de9977b187fcaabda585b015a568e905 2024-11-15T22:14:35,755 INFO [StoreOpener-de9977b187fcaabda585b015a568e905-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de9977b187fcaabda585b015a568e905 columnFamilyName family 2024-11-15T22:14:35,756 DEBUG [StoreOpener-de9977b187fcaabda585b015a568e905-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:35,757 INFO [StoreOpener-de9977b187fcaabda585b015a568e905-1 {}] regionserver.HStore(327): Store=de9977b187fcaabda585b015a568e905/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:14:35,757 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1038): replaying wal for de9977b187fcaabda585b015a568e905 2024-11-15T22:14:35,758 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/de9977b187fcaabda585b015a568e905 2024-11-15T22:14:35,759 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/de9977b187fcaabda585b015a568e905 2024-11-15T22:14:35,759 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1048): stopping wal replay for de9977b187fcaabda585b015a568e905 2024-11-15T22:14:35,760 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1060): Cleaning up temporary data for de9977b187fcaabda585b015a568e905 2024-11-15T22:14:35,761 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure ecaeddccb9b2987f43bb050f0a105976, server=e611192d6313,38729,1731708868111 because future has completed 2024-11-15T22:14:35,764 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): writing seq id for de9977b187fcaabda585b015a568e905 2024-11-15T22:14:35,771 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=10 2024-11-15T22:14:35,771 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure ecaeddccb9b2987f43bb050f0a105976, server=e611192d6313,38729,1731708868111 in 143 msec 2024-11-15T22:14:35,773 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/de9977b187fcaabda585b015a568e905/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T22:14:35,774 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1114): Opened de9977b187fcaabda585b015a568e905; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69468543, jitterRate=0.035161957144737244}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T22:14:35,774 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1122): Running coprocessor post-open hooks for de9977b187fcaabda585b015a568e905 2024-11-15T22:14:35,775 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1006): Region open journal for de9977b187fcaabda585b015a568e905: Running coprocessor pre-open hook at 1731708875747Writing region info on filesystem at 1731708875748 (+1 ms)Initializing all the Stores at 1731708875750 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708875750Cleaning up temporary data from old regions at 1731708875760 (+10 ms)Running coprocessor post-open hooks at 1731708875774 (+14 ms)Region opened successfully at 1731708875775 (+1 ms) 2024-11-15T22:14:35,777 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1731708873681.de9977b187fcaabda585b015a568e905., pid=50, masterSystemTime=1731708875689 2024-11-15T22:14:35,780 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=ecaeddccb9b2987f43bb050f0a105976, ASSIGN in 1.3510 sec 2024-11-15T22:14:35,783 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1731708873681.de9977b187fcaabda585b015a568e905. 2024-11-15T22:14:35,783 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(153): Opened test,lll,1731708873681.de9977b187fcaabda585b015a568e905. 2024-11-15T22:14:35,783 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(132): Open test,rrr,1731708873681.7502fc847b4b97e1ace9a22ec1c068e0. 2024-11-15T22:14:35,783 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7752): Opening region: {ENCODED => 7502fc847b4b97e1ace9a22ec1c068e0, NAME => 'test,rrr,1731708873681.7502fc847b4b97e1ace9a22ec1c068e0.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-15T22:14:35,784 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 7502fc847b4b97e1ace9a22ec1c068e0 2024-11-15T22:14:35,784 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(898): Instantiated test,rrr,1731708873681.7502fc847b4b97e1ace9a22ec1c068e0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:35,784 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7794): checking encryption for 7502fc847b4b97e1ace9a22ec1c068e0 2024-11-15T22:14:35,784 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7797): checking classloading for 7502fc847b4b97e1ace9a22ec1c068e0 2024-11-15T22:14:35,789 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=de9977b187fcaabda585b015a568e905, regionState=OPEN, openSeqNum=2, regionLocation=e611192d6313,38729,1731708868111 2024-11-15T22:14:35,789 INFO [StoreOpener-7502fc847b4b97e1ace9a22ec1c068e0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 7502fc847b4b97e1ace9a22ec1c068e0 2024-11-15T22:14:35,793 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure de9977b187fcaabda585b015a568e905, server=e611192d6313,38729,1731708868111 because future has completed 2024-11-15T22:14:35,793 INFO [StoreOpener-7502fc847b4b97e1ace9a22ec1c068e0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7502fc847b4b97e1ace9a22ec1c068e0 columnFamilyName family 2024-11-15T22:14:35,794 DEBUG [StoreOpener-7502fc847b4b97e1ace9a22ec1c068e0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:35,795 INFO [StoreOpener-7502fc847b4b97e1ace9a22ec1c068e0-1 {}] regionserver.HStore(327): Store=7502fc847b4b97e1ace9a22ec1c068e0/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:14:35,795 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1038): replaying wal for 7502fc847b4b97e1ace9a22ec1c068e0 2024-11-15T22:14:35,797 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/7502fc847b4b97e1ace9a22ec1c068e0 2024-11-15T22:14:35,800 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/7502fc847b4b97e1ace9a22ec1c068e0 2024-11-15T22:14:35,801 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1048): stopping wal replay for 7502fc847b4b97e1ace9a22ec1c068e0 2024-11-15T22:14:35,801 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1060): Cleaning up temporary data for 7502fc847b4b97e1ace9a22ec1c068e0 2024-11-15T22:14:35,803 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=16 2024-11-15T22:14:35,803 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure de9977b187fcaabda585b015a568e905, server=e611192d6313,38729,1731708868111 in 249 msec 2024-11-15T22:14:35,808 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=de9977b187fcaabda585b015a568e905, ASSIGN in 1.3830 sec 2024-11-15T22:14:35,808 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): writing seq id for 7502fc847b4b97e1ace9a22ec1c068e0 2024-11-15T22:14:35,814 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/7502fc847b4b97e1ace9a22ec1c068e0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T22:14:35,815 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1114): Opened 7502fc847b4b97e1ace9a22ec1c068e0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66003271, jitterRate=-0.016474619507789612}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T22:14:35,815 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7502fc847b4b97e1ace9a22ec1c068e0 2024-11-15T22:14:35,815 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1006): Region open journal for 7502fc847b4b97e1ace9a22ec1c068e0: Running coprocessor pre-open hook at 1731708875784Writing region info on filesystem at 1731708875784Initializing all the Stores at 1731708875788 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708875789 (+1 ms)Cleaning up temporary data from old regions at 1731708875801 (+12 ms)Running coprocessor post-open hooks at 1731708875815 (+14 ms)Region opened successfully at 1731708875815 2024-11-15T22:14:35,816 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1731708873681.7502fc847b4b97e1ace9a22ec1c068e0., pid=52, masterSystemTime=1731708875689 2024-11-15T22:14:35,820 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1731708873681.7502fc847b4b97e1ace9a22ec1c068e0. 2024-11-15T22:14:35,820 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(153): Opened test,rrr,1731708873681.7502fc847b4b97e1ace9a22ec1c068e0. 2024-11-15T22:14:35,820 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open test,ooo,1731708873681.a602f0e8bd42c685584b8b985174ddea. 2024-11-15T22:14:35,820 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => a602f0e8bd42c685584b8b985174ddea, NAME => 'test,ooo,1731708873681.a602f0e8bd42c685584b8b985174ddea.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-11-15T22:14:35,820 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=7502fc847b4b97e1ace9a22ec1c068e0, regionState=OPEN, openSeqNum=2, regionLocation=e611192d6313,38729,1731708868111 2024-11-15T22:14:35,821 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test a602f0e8bd42c685584b8b985174ddea 2024-11-15T22:14:35,821 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated test,ooo,1731708873681.a602f0e8bd42c685584b8b985174ddea.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:35,821 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for a602f0e8bd42c685584b8b985174ddea 2024-11-15T22:14:35,821 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for a602f0e8bd42c685584b8b985174ddea 2024-11-15T22:14:35,823 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39087 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=e611192d6313,38729,1731708868111, table=test, region=7502fc847b4b97e1ace9a22ec1c068e0. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T22:14:35,825 INFO [StoreOpener-a602f0e8bd42c685584b8b985174ddea-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region a602f0e8bd42c685584b8b985174ddea 2024-11-15T22:14:35,826 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7502fc847b4b97e1ace9a22ec1c068e0, server=e611192d6313,38729,1731708868111 because future has completed 2024-11-15T22:14:35,828 INFO [StoreOpener-a602f0e8bd42c685584b8b985174ddea-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a602f0e8bd42c685584b8b985174ddea columnFamilyName family 2024-11-15T22:14:35,828 DEBUG [StoreOpener-a602f0e8bd42c685584b8b985174ddea-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:35,831 INFO [StoreOpener-a602f0e8bd42c685584b8b985174ddea-1 {}] regionserver.HStore(327): Store=a602f0e8bd42c685584b8b985174ddea/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:14:35,831 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for a602f0e8bd42c685584b8b985174ddea 2024-11-15T22:14:35,832 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/a602f0e8bd42c685584b8b985174ddea 2024-11-15T22:14:35,834 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/a602f0e8bd42c685584b8b985174ddea 2024-11-15T22:14:35,835 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for a602f0e8bd42c685584b8b985174ddea 2024-11-15T22:14:35,835 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for a602f0e8bd42c685584b8b985174ddea 2024-11-15T22:14:35,839 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=22 2024-11-15T22:14:35,839 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure 7502fc847b4b97e1ace9a22ec1c068e0, server=e611192d6313,38729,1731708868111 in 261 msec 2024-11-15T22:14:35,843 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7502fc847b4b97e1ace9a22ec1c068e0, ASSIGN in 1.4190 sec 2024-11-15T22:14:35,848 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for a602f0e8bd42c685584b8b985174ddea 2024-11-15T22:14:35,857 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/a602f0e8bd42c685584b8b985174ddea/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T22:14:35,858 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened a602f0e8bd42c685584b8b985174ddea; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69025819, jitterRate=0.028564855456352234}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T22:14:35,859 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a602f0e8bd42c685584b8b985174ddea 2024-11-15T22:14:35,859 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for a602f0e8bd42c685584b8b985174ddea: Running coprocessor pre-open hook at 1731708875821Writing region info on filesystem at 1731708875821Initializing all the Stores at 1731708875825 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708875825Cleaning up temporary data from old regions at 1731708875835 (+10 ms)Running coprocessor post-open hooks at 1731708875859 (+24 ms)Region opened successfully at 1731708875859 2024-11-15T22:14:35,861 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1731708873681.a602f0e8bd42c685584b8b985174ddea., pid=51, masterSystemTime=1731708875689 2024-11-15T22:14:35,865 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ooo,1731708873681.a602f0e8bd42c685584b8b985174ddea. 2024-11-15T22:14:35,865 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened test,ooo,1731708873681.a602f0e8bd42c685584b8b985174ddea. 2024-11-15T22:14:35,865 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(132): Open test,yyy,1731708873681.c1cecce90107a4c37b7d91651975993f. 2024-11-15T22:14:35,866 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7752): Opening region: {ENCODED => c1cecce90107a4c37b7d91651975993f, NAME => 'test,yyy,1731708873681.c1cecce90107a4c37b7d91651975993f.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-15T22:14:35,867 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c1cecce90107a4c37b7d91651975993f 2024-11-15T22:14:35,867 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(898): Instantiated test,yyy,1731708873681.c1cecce90107a4c37b7d91651975993f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:35,867 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7794): checking encryption for c1cecce90107a4c37b7d91651975993f 2024-11-15T22:14:35,867 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7797): checking classloading for c1cecce90107a4c37b7d91651975993f 2024-11-15T22:14:35,869 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=a602f0e8bd42c685584b8b985174ddea, regionState=OPEN, openSeqNum=2, regionLocation=e611192d6313,38729,1731708868111 2024-11-15T22:14:35,876 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure a602f0e8bd42c685584b8b985174ddea, server=e611192d6313,38729,1731708868111 because future has completed 2024-11-15T22:14:35,880 INFO [StoreOpener-c1cecce90107a4c37b7d91651975993f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c1cecce90107a4c37b7d91651975993f 2024-11-15T22:14:35,889 INFO [StoreOpener-c1cecce90107a4c37b7d91651975993f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1cecce90107a4c37b7d91651975993f columnFamilyName family 2024-11-15T22:14:35,889 DEBUG [StoreOpener-c1cecce90107a4c37b7d91651975993f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:35,891 INFO [StoreOpener-c1cecce90107a4c37b7d91651975993f-1 {}] regionserver.HStore(327): Store=c1cecce90107a4c37b7d91651975993f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:14:35,891 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1038): replaying wal for c1cecce90107a4c37b7d91651975993f 2024-11-15T22:14:35,893 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/c1cecce90107a4c37b7d91651975993f 2024-11-15T22:14:35,894 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/c1cecce90107a4c37b7d91651975993f 2024-11-15T22:14:35,897 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1048): stopping wal replay for c1cecce90107a4c37b7d91651975993f 2024-11-15T22:14:35,897 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1060): Cleaning up temporary data for c1cecce90107a4c37b7d91651975993f 2024-11-15T22:14:35,899 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): writing seq id for c1cecce90107a4c37b7d91651975993f 2024-11-15T22:14:35,904 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=19 2024-11-15T22:14:35,904 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/c1cecce90107a4c37b7d91651975993f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T22:14:35,905 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=19, state=SUCCESS, hasLock=false; OpenRegionProcedure a602f0e8bd42c685584b8b985174ddea, server=e611192d6313,38729,1731708868111 in 319 msec 2024-11-15T22:14:35,907 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1114): Opened c1cecce90107a4c37b7d91651975993f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58988180, jitterRate=-0.12100762128829956}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T22:14:35,907 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c1cecce90107a4c37b7d91651975993f 2024-11-15T22:14:35,907 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1006): Region open journal for c1cecce90107a4c37b7d91651975993f: Running coprocessor pre-open hook at 1731708875867Writing region info on filesystem at 1731708875867Initializing all the Stores at 1731708875872 (+5 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708875872Cleaning up temporary data from old regions at 1731708875897 (+25 ms)Running coprocessor post-open hooks at 1731708875907 (+10 ms)Region opened successfully at 1731708875907 2024-11-15T22:14:35,912 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1731708873681.c1cecce90107a4c37b7d91651975993f., pid=49, masterSystemTime=1731708875689 2024-11-15T22:14:35,917 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a602f0e8bd42c685584b8b985174ddea, ASSIGN in 1.4850 sec 2024-11-15T22:14:35,921 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1731708873681.c1cecce90107a4c37b7d91651975993f. 2024-11-15T22:14:35,922 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(153): Opened test,yyy,1731708873681.c1cecce90107a4c37b7d91651975993f. 2024-11-15T22:14:35,922 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(132): Open test,iii,1731708873681.1732b30f3d981289760d152f788aff6f. 2024-11-15T22:14:35,922 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7752): Opening region: {ENCODED => 1732b30f3d981289760d152f788aff6f, NAME => 'test,iii,1731708873681.1732b30f3d981289760d152f788aff6f.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-15T22:14:35,922 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 1732b30f3d981289760d152f788aff6f 2024-11-15T22:14:35,923 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(898): Instantiated test,iii,1731708873681.1732b30f3d981289760d152f788aff6f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:35,923 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7794): checking encryption for 1732b30f3d981289760d152f788aff6f 2024-11-15T22:14:35,923 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7797): checking classloading for 1732b30f3d981289760d152f788aff6f 2024-11-15T22:14:35,925 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=c1cecce90107a4c37b7d91651975993f, regionState=OPEN, openSeqNum=2, regionLocation=e611192d6313,38729,1731708868111 2024-11-15T22:14:35,927 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39087 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=e611192d6313,38729,1731708868111, table=test, region=c1cecce90107a4c37b7d91651975993f. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T22:14:35,928 INFO [StoreOpener-1732b30f3d981289760d152f788aff6f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 1732b30f3d981289760d152f788aff6f 2024-11-15T22:14:35,929 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure c1cecce90107a4c37b7d91651975993f, server=e611192d6313,38729,1731708868111 because future has completed 2024-11-15T22:14:35,933 INFO [StoreOpener-1732b30f3d981289760d152f788aff6f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1732b30f3d981289760d152f788aff6f columnFamilyName family 2024-11-15T22:14:35,933 DEBUG [StoreOpener-1732b30f3d981289760d152f788aff6f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:35,934 INFO [StoreOpener-1732b30f3d981289760d152f788aff6f-1 {}] regionserver.HStore(327): Store=1732b30f3d981289760d152f788aff6f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:14:35,934 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1038): replaying wal for 1732b30f3d981289760d152f788aff6f 2024-11-15T22:14:35,936 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/1732b30f3d981289760d152f788aff6f 2024-11-15T22:14:35,939 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=29 2024-11-15T22:14:35,939 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure c1cecce90107a4c37b7d91651975993f, server=e611192d6313,38729,1731708868111 in 399 msec 2024-11-15T22:14:35,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T22:14:35,943 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/1732b30f3d981289760d152f788aff6f 2024-11-15T22:14:35,943 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1048): stopping wal replay for 1732b30f3d981289760d152f788aff6f 2024-11-15T22:14:35,944 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1060): Cleaning up temporary data for 1732b30f3d981289760d152f788aff6f 2024-11-15T22:14:35,946 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=c1cecce90107a4c37b7d91651975993f, ASSIGN in 1.5190 sec 2024-11-15T22:14:35,946 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1093): writing seq id for 1732b30f3d981289760d152f788aff6f 2024-11-15T22:14:35,949 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/1732b30f3d981289760d152f788aff6f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T22:14:35,950 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1114): Opened 1732b30f3d981289760d152f788aff6f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63023659, jitterRate=-0.060874298214912415}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T22:14:35,950 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1732b30f3d981289760d152f788aff6f 2024-11-15T22:14:35,950 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1006): Region open journal for 1732b30f3d981289760d152f788aff6f: Running coprocessor pre-open hook at 1731708875923Writing region info on filesystem at 1731708875923Initializing all the Stores at 1731708875925 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708875925Cleaning up temporary data from old regions at 1731708875944 (+19 ms)Running coprocessor post-open hooks at 1731708875950 (+6 ms)Region opened successfully at 1731708875950 2024-11-15T22:14:35,952 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1731708873681.1732b30f3d981289760d152f788aff6f., pid=54, masterSystemTime=1731708875689 2024-11-15T22:14:35,957 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1731708873681.1732b30f3d981289760d152f788aff6f. 2024-11-15T22:14:35,957 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(153): Opened test,iii,1731708873681.1732b30f3d981289760d152f788aff6f. 2024-11-15T22:14:35,958 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open test,ccc,1731708873681.2424ff4b6d907aafacef6f661d88ef95. 2024-11-15T22:14:35,958 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => 2424ff4b6d907aafacef6f661d88ef95, NAME => 'test,ccc,1731708873681.2424ff4b6d907aafacef6f661d88ef95.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-15T22:14:35,958 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 2424ff4b6d907aafacef6f661d88ef95 2024-11-15T22:14:35,958 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated test,ccc,1731708873681.2424ff4b6d907aafacef6f661d88ef95.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:35,959 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for 2424ff4b6d907aafacef6f661d88ef95 2024-11-15T22:14:35,959 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for 2424ff4b6d907aafacef6f661d88ef95 2024-11-15T22:14:35,960 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=1732b30f3d981289760d152f788aff6f, regionState=OPEN, openSeqNum=2, regionLocation=e611192d6313,38729,1731708868111 2024-11-15T22:14:35,964 INFO [StoreOpener-2424ff4b6d907aafacef6f661d88ef95-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 2424ff4b6d907aafacef6f661d88ef95 2024-11-15T22:14:35,965 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1732b30f3d981289760d152f788aff6f, server=e611192d6313,38729,1731708868111 because future has completed 2024-11-15T22:14:35,967 INFO [StoreOpener-2424ff4b6d907aafacef6f661d88ef95-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2424ff4b6d907aafacef6f661d88ef95 columnFamilyName family 2024-11-15T22:14:35,967 DEBUG [StoreOpener-2424ff4b6d907aafacef6f661d88ef95-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:35,968 INFO [StoreOpener-2424ff4b6d907aafacef6f661d88ef95-1 {}] regionserver.HStore(327): Store=2424ff4b6d907aafacef6f661d88ef95/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:14:35,968 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for 2424ff4b6d907aafacef6f661d88ef95 2024-11-15T22:14:35,969 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/2424ff4b6d907aafacef6f661d88ef95 2024-11-15T22:14:35,970 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/2424ff4b6d907aafacef6f661d88ef95 2024-11-15T22:14:35,974 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for 2424ff4b6d907aafacef6f661d88ef95 2024-11-15T22:14:35,974 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for 2424ff4b6d907aafacef6f661d88ef95 2024-11-15T22:14:35,977 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for 2424ff4b6d907aafacef6f661d88ef95 2024-11-15T22:14:35,980 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/2424ff4b6d907aafacef6f661d88ef95/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T22:14:35,980 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened 2424ff4b6d907aafacef6f661d88ef95; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72139382, jitterRate=0.07496055960655212}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T22:14:35,981 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2424ff4b6d907aafacef6f661d88ef95 2024-11-15T22:14:35,981 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for 2424ff4b6d907aafacef6f661d88ef95: Running coprocessor pre-open hook at 1731708875959Writing region info on filesystem at 1731708875959Initializing all the Stores at 1731708875960 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708875960Cleaning up temporary data from old regions at 1731708875974 (+14 ms)Running coprocessor post-open hooks at 1731708875981 (+7 ms)Region opened successfully at 1731708875981 2024-11-15T22:14:35,984 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1731708873681.2424ff4b6d907aafacef6f661d88ef95., pid=55, masterSystemTime=1731708875689 2024-11-15T22:14:35,984 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=13 2024-11-15T22:14:35,985 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure 1732b30f3d981289760d152f788aff6f, server=e611192d6313,38729,1731708868111 in 372 msec 2024-11-15T22:14:35,991 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1731708873681.2424ff4b6d907aafacef6f661d88ef95. 2024-11-15T22:14:35,991 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened test,ccc,1731708873681.2424ff4b6d907aafacef6f661d88ef95. 2024-11-15T22:14:35,991 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open test,vvv,1731708873681.f9e6155d6c82d9d9a5c9449be6d7ce5f. 2024-11-15T22:14:35,992 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => f9e6155d6c82d9d9a5c9449be6d7ce5f, NAME => 'test,vvv,1731708873681.f9e6155d6c82d9d9a5c9449be6d7ce5f.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-15T22:14:35,992 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=1732b30f3d981289760d152f788aff6f, ASSIGN in 1.5650 sec 2024-11-15T22:14:35,992 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f9e6155d6c82d9d9a5c9449be6d7ce5f 2024-11-15T22:14:35,993 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated test,vvv,1731708873681.f9e6155d6c82d9d9a5c9449be6d7ce5f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:35,993 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for f9e6155d6c82d9d9a5c9449be6d7ce5f 2024-11-15T22:14:35,993 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for f9e6155d6c82d9d9a5c9449be6d7ce5f 2024-11-15T22:14:35,997 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=2424ff4b6d907aafacef6f661d88ef95, regionState=OPEN, openSeqNum=2, regionLocation=e611192d6313,38729,1731708868111 2024-11-15T22:14:36,001 INFO [StoreOpener-f9e6155d6c82d9d9a5c9449be6d7ce5f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f9e6155d6c82d9d9a5c9449be6d7ce5f 2024-11-15T22:14:36,001 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2424ff4b6d907aafacef6f661d88ef95, server=e611192d6313,38729,1731708868111 because future has completed 2024-11-15T22:14:36,004 INFO [StoreOpener-f9e6155d6c82d9d9a5c9449be6d7ce5f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f9e6155d6c82d9d9a5c9449be6d7ce5f columnFamilyName family 2024-11-15T22:14:36,004 DEBUG [StoreOpener-f9e6155d6c82d9d9a5c9449be6d7ce5f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:36,005 INFO [StoreOpener-f9e6155d6c82d9d9a5c9449be6d7ce5f-1 {}] regionserver.HStore(327): Store=f9e6155d6c82d9d9a5c9449be6d7ce5f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:14:36,005 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for f9e6155d6c82d9d9a5c9449be6d7ce5f 2024-11-15T22:14:36,010 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=7 2024-11-15T22:14:36,010 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 2424ff4b6d907aafacef6f661d88ef95, server=e611192d6313,38729,1731708868111 in 389 msec 2024-11-15T22:14:36,011 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/f9e6155d6c82d9d9a5c9449be6d7ce5f 2024-11-15T22:14:36,013 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/f9e6155d6c82d9d9a5c9449be6d7ce5f 2024-11-15T22:14:36,013 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for f9e6155d6c82d9d9a5c9449be6d7ce5f 2024-11-15T22:14:36,014 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for f9e6155d6c82d9d9a5c9449be6d7ce5f 2024-11-15T22:14:36,016 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=2424ff4b6d907aafacef6f661d88ef95, ASSIGN in 1.5910 sec 2024-11-15T22:14:36,017 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for f9e6155d6c82d9d9a5c9449be6d7ce5f 2024-11-15T22:14:36,020 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/f9e6155d6c82d9d9a5c9449be6d7ce5f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T22:14:36,021 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened f9e6155d6c82d9d9a5c9449be6d7ce5f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73139271, jitterRate=0.08986006677150726}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T22:14:36,021 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f9e6155d6c82d9d9a5c9449be6d7ce5f 2024-11-15T22:14:36,021 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for f9e6155d6c82d9d9a5c9449be6d7ce5f: Running coprocessor pre-open hook at 1731708875993Writing region info on filesystem at 1731708875993Initializing all the Stores at 1731708875999 (+6 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708876000 (+1 ms)Cleaning up temporary data from old regions at 1731708876014 (+14 ms)Running coprocessor post-open hooks at 1731708876021 (+7 ms)Region opened successfully at 1731708876021 2024-11-15T22:14:36,024 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1731708873681.f9e6155d6c82d9d9a5c9449be6d7ce5f., pid=53, masterSystemTime=1731708875689 2024-11-15T22:14:36,029 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1731708873681.f9e6155d6c82d9d9a5c9449be6d7ce5f. 2024-11-15T22:14:36,029 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened test,vvv,1731708873681.f9e6155d6c82d9d9a5c9449be6d7ce5f. 2024-11-15T22:14:36,034 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=f9e6155d6c82d9d9a5c9449be6d7ce5f, regionState=OPEN, openSeqNum=2, regionLocation=e611192d6313,38729,1731708868111 2024-11-15T22:14:36,037 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure f9e6155d6c82d9d9a5c9449be6d7ce5f, server=e611192d6313,38729,1731708868111 because future has completed 2024-11-15T22:14:36,051 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=26 2024-11-15T22:14:36,051 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure f9e6155d6c82d9d9a5c9449be6d7ce5f, server=e611192d6313,38729,1731708868111 in 465 msec 2024-11-15T22:14:36,060 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=4 2024-11-15T22:14:36,060 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f9e6155d6c82d9d9a5c9449be6d7ce5f, ASSIGN in 1.6310 sec 2024-11-15T22:14:36,063 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T22:14:36,063 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731708876063"}]},"ts":"1731708876063"} 2024-11-15T22:14:36,071 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLED in hbase:meta 2024-11-15T22:14:36,076 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T22:14:36,089 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test in 2.3850 sec 2024-11-15T22:14:37,616 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-15T22:14:37,617 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-15T22:14:37,624 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test 2024-11-15T22:14:37,624 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test Metrics about Tables on a single HBase RegionServer 2024-11-15T22:14:37,626 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-15T22:14:37,627 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-15T22:14:37,627 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-15T22:14:37,627 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-15T22:14:37,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T22:14:37,949 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table test get assigned. Timeout = 60000ms 2024-11-15T22:14:37,949 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test completed 2024-11-15T22:14:37,951 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-15T22:14:37,972 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table test assigned to meta. Checking AM states. 2024-11-15T22:14:37,973 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-15T22:14:37,974 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table test assigned. 2024-11-15T22:14:37,974 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-15T22:14:37,982 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-15T22:14:38,015 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-15T22:14:38,052 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'test', row='', locateType=CURRENT is [region=test,,1731708873681.f2783381a48c13c2f75c07e5029903cb., hostname=e611192d6313,43349,1731708869385, seqNum=2] 2024-11-15T22:14:38,061 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:14:38,073 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56892, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:14:38,076 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:14:38,085 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43536, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:14:38,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39087 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 2424ff4b6d907aafacef6f661d88ef95 changed from -1.0 to 0.0, refreshing cache 2024-11-15T22:14:38,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39087 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ecaeddccb9b2987f43bb050f0a105976 changed from -1.0 to 0.0, refreshing cache 2024-11-15T22:14:38,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39087 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1732b30f3d981289760d152f788aff6f changed from -1.0 to 0.0, refreshing cache 2024-11-15T22:14:38,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39087 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region de9977b187fcaabda585b015a568e905 changed from -1.0 to 0.0, refreshing cache 2024-11-15T22:14:38,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39087 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region a602f0e8bd42c685584b8b985174ddea changed from -1.0 to 0.0, refreshing cache 2024-11-15T22:14:38,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39087 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 7502fc847b4b97e1ace9a22ec1c068e0 changed from -1.0 to 0.0, refreshing cache 2024-11-15T22:14:38,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39087 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f9e6155d6c82d9d9a5c9449be6d7ce5f changed from -1.0 to 0.0, refreshing cache 2024-11-15T22:14:38,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39087 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c1cecce90107a4c37b7d91651975993f changed from -1.0 to 0.0, refreshing cache 2024-11-15T22:14:38,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39087 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 6efdb15046141ab0cd574540ef2ac2c2 changed from -1.0 to 0.0, refreshing cache 2024-11-15T22:14:38,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39087 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 51bf57bcf1afa9f625176363284d17f7 changed from -1.0 to 0.0, refreshing cache 2024-11-15T22:14:38,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39087 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1959cb95bb21ecdc677f7f5290173984 changed from -1.0 to 0.0, refreshing cache 2024-11-15T22:14:38,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39087 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 36b8b3bed3adb94f5901f1f3809b64fa changed from -1.0 to 0.0, refreshing cache 2024-11-15T22:14:38,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39087 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f64d8a749084ba25ea1b77db9a0d9f17 changed from -1.0 to 0.0, refreshing cache 2024-11-15T22:14:38,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39087 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 83048e0be415b190a19f64bc2e49d3fe changed from -1.0 to 0.0, refreshing cache 2024-11-15T22:14:38,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39087 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 4381d14830a288bee3aeb42195dd8768 changed from -1.0 to 0.0, refreshing cache 2024-11-15T22:14:38,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39087 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region e87a82b283b7251342a664ba3876c98e changed from -1.0 to 0.0, refreshing cache 2024-11-15T22:14:38,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39087 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region a7fb991e3993a690fe7a0a1b064d5f5d changed from -1.0 to 0.0, refreshing cache 2024-11-15T22:14:38,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39087 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f2783381a48c13c2f75c07e5029903cb changed from -1.0 to 0.0, refreshing cache 2024-11-15T22:14:38,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39087 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 897a7bb57a002d365489ce583eb7f84f changed from -1.0 to 0.0, refreshing cache 2024-11-15T22:14:38,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39087 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 93deaad3e3229244196e8daa9478371a changed from -1.0 to 0.0, refreshing cache 2024-11-15T22:14:38,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39087 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 13c3457b9645c9a8181ed9b4551962ad changed from -1.0 to 0.0, refreshing cache 2024-11-15T22:14:38,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39087 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 2a159d388ea2138ee8e2bed061a785ce changed from -1.0 to 0.0, refreshing cache 2024-11-15T22:14:38,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39087 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c56d152c07a95a189ac65341ff4db1fa changed from -1.0 to 0.0, refreshing cache 2024-11-15T22:14:38,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39087 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 06b9df8f55d791077e9f7ec55ef6a971 changed from -1.0 to 0.0, refreshing cache 2024-11-15T22:14:38,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39087 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 819421a6406c0ebb5635c399b2c0303d changed from -1.0 to 0.0, refreshing cache 2024-11-15T22:14:38,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39087 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 2e79a8ce6f9715045d056d3dfdef5cec changed from -1.0 to 0.0, refreshing cache 2024-11-15T22:14:38,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39087 {}] rsgroup.RSGroupBasedLoadBalancer(138): Balancing RSGroup=default 2024-11-15T22:14:38,162 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39087 {}] rsgroup.RSGroupBasedLoadBalancer(151): Start Generate Balance plan for group: default 2024-11-15T22:14:38,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39087 {}] balancer.BaseLoadBalancer(619): Start Generate Balance plan for cluster. 2024-11-15T22:14:38,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39087 {}] balancer.BalancerClusterState(204): Hosts are {e611192d6313=0} racks are {/default-rack=0} 2024-11-15T22:14:38,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39087 {}] balancer.BalancerClusterState(303): server 0 has 9 regions 2024-11-15T22:14:38,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39087 {}] balancer.BalancerClusterState(303): server 1 has 8 regions 2024-11-15T22:14:38,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39087 {}] balancer.BalancerClusterState(303): server 2 has 10 regions 2024-11-15T22:14:38,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39087 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-15T22:14:38,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39087 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-15T22:14:38,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39087 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-15T22:14:38,163 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39087 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-15T22:14:38,163 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39087 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-15T22:14:38,163 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39087 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-15T22:14:38,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39087 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1 2024-11-15T22:14:38,169 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39087 {}] balancer.StochasticLoadBalancer(403): Cluster wide - Calculating plan. may take up to 30000ms to complete. 2024-11-15T22:14:38,170 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39087 {}] balancer.StochasticLoadBalancer(515): Start StochasticLoadBalancer.balancer, initial weighted average imbalance=0.06931633435746164, functionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.06415002990995843, need balance); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.0); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); computedMaxSteps=64800 2024-11-15T22:14:38,796 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T22:14:38,867 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-15T22:14:38,870 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test' 2024-11-15T22:14:38,986 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39087 {}] balancer.StochasticLoadBalancer(562): Finished computing new moving plan. Computation took 822 ms to try 64800 different iterations. Found a solution that moves 1 regions; Going from a computed imbalance of 0.06931633435746164 to a new imbalance of 0.01690158032826896. funtionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.0); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.037037037037037035, need balance); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); 2024-11-15T22:14:38,991 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39087 {}] master.HMaster(2167): Balancer plans size is 1, the balance interval is 300000 ms, and the max number regions in transition is 27 2024-11-15T22:14:38,991 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39087 {}] master.HMaster(2172): balance hri=93deaad3e3229244196e8daa9478371a, source=e611192d6313,43349,1731708869385, destination=e611192d6313,41231,1731708869041 2024-11-15T22:14:38,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39087 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test, region=93deaad3e3229244196e8daa9478371a, REOPEN/MOVE 2024-11-15T22:14:38,997 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test, region=93deaad3e3229244196e8daa9478371a, REOPEN/MOVE 2024-11-15T22:14:39,000 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=93deaad3e3229244196e8daa9478371a, regionState=CLOSING, regionLocation=e611192d6313,43349,1731708869385 2024-11-15T22:14:39,005 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=test, region=93deaad3e3229244196e8daa9478371a, REOPEN/MOVE because future has completed 2024-11-15T22:14:39,006 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T22:14:39,006 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE, hasLock=false; CloseRegionProcedure 93deaad3e3229244196e8daa9478371a, server=e611192d6313,43349,1731708869385}] 2024-11-15T22:14:39,165 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(122): Close 93deaad3e3229244196e8daa9478371a 2024-11-15T22:14:39,165 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T22:14:39,166 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1722): Closing 93deaad3e3229244196e8daa9478371a, disabling compactions & flushes 2024-11-15T22:14:39,166 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1755): Closing region test,eee,1731708873681.93deaad3e3229244196e8daa9478371a. 2024-11-15T22:14:39,166 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1731708873681.93deaad3e3229244196e8daa9478371a. 2024-11-15T22:14:39,166 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1843): Acquired close lock on test,eee,1731708873681.93deaad3e3229244196e8daa9478371a. after waiting 0 ms 2024-11-15T22:14:39,166 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1853): Updates disabled for region test,eee,1731708873681.93deaad3e3229244196e8daa9478371a. 2024-11-15T22:14:39,185 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/93deaad3e3229244196e8daa9478371a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T22:14:39,189 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1973): Closed test,eee,1731708873681.93deaad3e3229244196e8daa9478371a. 2024-11-15T22:14:39,189 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1676): Region close journal for 93deaad3e3229244196e8daa9478371a: Waiting for close lock at 1731708879166Running coprocessor pre-close hooks at 1731708879166Disabling compacts and flushes for region at 1731708879166Disabling writes for close at 1731708879166Writing region close event to WAL at 1731708879172 (+6 ms)Running coprocessor post-close hooks at 1731708879186 (+14 ms)Closed at 1731708879189 (+3 ms) 2024-11-15T22:14:39,189 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegionServer(3302): Adding 93deaad3e3229244196e8daa9478371a move to e611192d6313,41231,1731708869041 record at close sequenceid=2 2024-11-15T22:14:39,198 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(157): Closed 93deaad3e3229244196e8daa9478371a 2024-11-15T22:14:39,199 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=93deaad3e3229244196e8daa9478371a, regionState=CLOSED 2024-11-15T22:14:39,209 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE, hasLock=false; CloseRegionProcedure 93deaad3e3229244196e8daa9478371a, server=e611192d6313,43349,1731708869385 because future has completed 2024-11-15T22:14:39,214 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-15T22:14:39,214 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; CloseRegionProcedure 93deaad3e3229244196e8daa9478371a, server=e611192d6313,43349,1731708869385 in 205 msec 2024-11-15T22:14:39,216 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=93deaad3e3229244196e8daa9478371a, REOPEN/MOVE; state=CLOSED, location=e611192d6313,41231,1731708869041; forceNewPlan=false, retain=false 2024-11-15T22:14:39,367 INFO [e611192d6313:39087 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-15T22:14:39,367 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=93deaad3e3229244196e8daa9478371a, regionState=OPENING, regionLocation=e611192d6313,41231,1731708869041 2024-11-15T22:14:39,370 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=93deaad3e3229244196e8daa9478371a, REOPEN/MOVE because future has completed 2024-11-15T22:14:39,371 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=57, state=RUNNABLE, hasLock=false; OpenRegionProcedure 93deaad3e3229244196e8daa9478371a, server=e611192d6313,41231,1731708869041}] 2024-11-15T22:14:39,528 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(132): Open test,eee,1731708873681.93deaad3e3229244196e8daa9478371a. 2024-11-15T22:14:39,528 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7752): Opening region: {ENCODED => 93deaad3e3229244196e8daa9478371a, NAME => 'test,eee,1731708873681.93deaad3e3229244196e8daa9478371a.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-15T22:14:39,529 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 93deaad3e3229244196e8daa9478371a 2024-11-15T22:14:39,529 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(898): Instantiated test,eee,1731708873681.93deaad3e3229244196e8daa9478371a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:39,529 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7794): checking encryption for 93deaad3e3229244196e8daa9478371a 2024-11-15T22:14:39,529 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7797): checking classloading for 93deaad3e3229244196e8daa9478371a 2024-11-15T22:14:39,531 INFO [StoreOpener-93deaad3e3229244196e8daa9478371a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 93deaad3e3229244196e8daa9478371a 2024-11-15T22:14:39,532 INFO [StoreOpener-93deaad3e3229244196e8daa9478371a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 93deaad3e3229244196e8daa9478371a columnFamilyName family 2024-11-15T22:14:39,532 DEBUG [StoreOpener-93deaad3e3229244196e8daa9478371a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:39,532 INFO [StoreOpener-93deaad3e3229244196e8daa9478371a-1 {}] regionserver.HStore(327): Store=93deaad3e3229244196e8daa9478371a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:14:39,533 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1038): replaying wal for 93deaad3e3229244196e8daa9478371a 2024-11-15T22:14:39,534 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/93deaad3e3229244196e8daa9478371a 2024-11-15T22:14:39,535 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/93deaad3e3229244196e8daa9478371a 2024-11-15T22:14:39,535 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1048): stopping wal replay for 93deaad3e3229244196e8daa9478371a 2024-11-15T22:14:39,536 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1060): Cleaning up temporary data for 93deaad3e3229244196e8daa9478371a 2024-11-15T22:14:39,539 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1093): writing seq id for 93deaad3e3229244196e8daa9478371a 2024-11-15T22:14:39,540 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1114): Opened 93deaad3e3229244196e8daa9478371a; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74435010, jitterRate=0.10916808247566223}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T22:14:39,540 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 93deaad3e3229244196e8daa9478371a 2024-11-15T22:14:39,540 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1006): Region open journal for 93deaad3e3229244196e8daa9478371a: Running coprocessor pre-open hook at 1731708879529Writing region info on filesystem at 1731708879529Initializing all the Stores at 1731708879530 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708879530Cleaning up temporary data from old regions at 1731708879536 (+6 ms)Running coprocessor post-open hooks at 1731708879540 (+4 ms)Region opened successfully at 1731708879540 2024-11-15T22:14:39,542 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1731708873681.93deaad3e3229244196e8daa9478371a., pid=59, masterSystemTime=1731708879523 2024-11-15T22:14:39,544 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1731708873681.93deaad3e3229244196e8daa9478371a. 2024-11-15T22:14:39,545 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(153): Opened test,eee,1731708873681.93deaad3e3229244196e8daa9478371a. 2024-11-15T22:14:39,545 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=93deaad3e3229244196e8daa9478371a, regionState=OPEN, openSeqNum=5, regionLocation=e611192d6313,41231,1731708869041 2024-11-15T22:14:39,549 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39087 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=e611192d6313,41231,1731708869041, table=test, region=93deaad3e3229244196e8daa9478371a. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T22:14:39,549 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=57, state=RUNNABLE, hasLock=false; OpenRegionProcedure 93deaad3e3229244196e8daa9478371a, server=e611192d6313,41231,1731708869041 because future has completed 2024-11-15T22:14:39,554 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=57 2024-11-15T22:14:39,554 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=57, state=SUCCESS, hasLock=false; OpenRegionProcedure 93deaad3e3229244196e8daa9478371a, server=e611192d6313,41231,1731708869041 in 181 msec 2024-11-15T22:14:39,566 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=93deaad3e3229244196e8daa9478371a, REOPEN/MOVE in 563 msec 2024-11-15T22:14:39,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39087 {}] master.HMaster(2203): Balancer is going into sleep until next period in 300000ms 2024-11-15T22:14:39,603 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [900,000] milli-secs(wait.for.ratio=[1]) 2024-11-15T22:14:39,612 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39087 {}] master.MasterRpcServices(567): Client=jenkins//172.17.0.3 set balanceSwitch=false 2024-11-15T22:14:39,681 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.assignment.TestRollbackSCP#testFailAndRollback Thread=377, OpenFileDescriptor=635, MaxFileDescriptor=1048576, SystemLoadAverage=971, ProcessCount=11, AvailableMemoryMB=265 2024-11-15T22:14:39,684 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table hbase:meta 2024-11-15T22:14:39,685 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=hbase:meta,,1 2024-11-15T22:14:39,685 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing e611192d6313,43349,1731708869385 2024-11-15T22:14:39,687 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server e611192d6313,43349,1731708869385: Simulated kill ***** 2024-11-15T22:14:39,687 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-15T22:14:39,690 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-15T22:14:39,698 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-15T22:14:39,702 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-15T22:14:39,704 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-15T22:14:39,714 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 361578472 }, "NonHeapMemoryUsage": { "committed": 155058176, "init": 7667712, "max": -1, "used": 152751176 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "e611192d6313", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 2, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 34393, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 100, "ProcessCallTime_min": 0, "ProcessCallTime_max": 28, "ProcessCallTime_mean": 4, "ProcessCallTime_25th_percentile": 7, "ProcessCallTime_median": 14, "ProcessCallTime_75th_percentile": 21, "ProcessCallTime_90th_percentile": 25, "ProcessCallTime_95th_percentile": 26, "ProcessCallTime_98th_percentile": 27, "ProcessCallTime_99th_percentile": 27, "ProcessCallTime_99.9th_percentile": 27, "ProcessCallTime_TimeRangeCount_0-1": 100, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 100, "QueueCallTime_min": 0, "QueueCallTime_max": 67, "QueueCallTime_mean": 7, "QueueCallTime_25th_percentile": 16, "QueueCallTime_median": 33, "QueueCallTime_75th_percentile": 50, "QueueCallTime_90th_percentile": 60, "QueueCallTime_95th_percentile": 63, "QueueCallTime_98th_percentile": 65, "QueueCallTime_99th_percentile": 66, "QueueCallTime_99.9th_percentile": 66, "QueueCallTime_TimeRangeCount_0-1": 100, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 100, "TotalCallTime_min": 0, "TotalCallTime_max": 72, "TotalCallTime_mean": 12, "TotalCallTime_25th_percentile": 18, "TotalCallTime_median": 36, "TotalCallTime_75th_percentile": 54, "TotalCallTime_90th_percentile": 64, "TotalCallTime_95th_percentile": 68, "TotalCallTime_98th_percentile": 70, "TotalCallTime_99th_percentile": 71, "TotalCallTime_99.9th_percentile": 71, "TotalCallTime_TimeRangeCount_0-1": 100, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 100, "ResponseSize_min": 0, "ResponseSize_max": 22359, "ResponseSize_mean": 692, "ResponseSize_25th_percentile": 5589, "ResponseSize_median": 11179, "ResponseSize_75th_percentile": 16769, "ResponseSize_90th_percentile": 20123, "ResponseSize_95th_percentile": 21241, "ResponseSize_98th_percentile": 21911, "ResponseSize_99th_percentile": 22135, "ResponseSize_99.9th_percentile": 22336, "ResponseSize_SizeRangeCount_0-10": 100, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 100, "RequestSize_min": 81, "RequestSize_max": 8095, "RequestSize_mean": 311, "RequestSize_25th_percentile": 2084, "RequestSize_median": 4088, "RequestSize_75th_percentile": 6091, "RequestSize_90th_percentile": 7293, "RequestSize_95th_percentile": 7694, "RequestSize_98th_percentile": 7934, "RequestSize_99th_percentile": 8014, "RequestSize_99.9th_percentile": 8086, "RequestSize_SizeRangeCount_0-10": 100, "sentBytes": 55432 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "e611192d6313", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:60621", "tag.serverName": "e611192d6313,41231,1731708869041", "tag.clusterId": "1b4a912b-fe9b-4940-8b4d-31902347eb12", "tag.Context": "regionserver", "tag.Hostname": "e611192d6313", "regionCount": 8, "storeCount": 8, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 2048, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1731708869041, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 0, "localBytesRead": 0, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 0, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 8, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 8, "rpcScanRequestCount": 8, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 8, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "ScanTime_TimeRangeCount_0-1": 8, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 8, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "ScanSize_SizeRangeCount_0-10": 8, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-15T22:14:39,719 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39087 {}] master.MasterRpcServices(700): e611192d6313,43349,1731708869385 reported a fatal error: ***** ABORTING region server e611192d6313,43349,1731708869385: Simulated kill ***** 2024-11-15T22:14:39,722 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'e611192d6313,43349,1731708869385' ***** 2024-11-15T22:14:39,722 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-11-15T22:14:39,722 INFO [RS:2;e611192d6313:43349 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-15T22:14:39,722 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-15T22:14:39,722 INFO [RS:2;e611192d6313:43349 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-15T22:14:39,722 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [15,000] milli-secs(wait.for.ratio=[1]) 2024-11-15T22:14:39,722 INFO [RS:2;e611192d6313:43349 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-15T22:14:39,722 INFO [RS:2;e611192d6313:43349 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-15T22:14:39,723 INFO [RS:2;e611192d6313:43349 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;e611192d6313:43349. 2024-11-15T22:14:39,723 DEBUG [RS:2;e611192d6313:43349 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T22:14:39,728 DEBUG [RS:2;e611192d6313:43349 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:14:39,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33631 is added to blk_1073741836_1012 (size=33749) 2024-11-15T22:14:39,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39269 is added to blk_1073741836_1012 (size=33749) 2024-11-15T22:14:39,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741836_1012 (size=33749) 2024-11-15T22:14:39,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741833_1009 (size=2769) 2024-11-15T22:14:39,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39269 is added to blk_1073741833_1009 (size=2769) 2024-11-15T22:14:39,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33631 is added to blk_1073741833_1009 (size=2769) 2024-11-15T22:14:39,744 DEBUG [RS:2;e611192d6313:43349 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:14:39,744 INFO [RS:2;e611192d6313:43349 {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T22:14:39,745 INFO [RS:2;e611192d6313:43349 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:43349 2024-11-15T22:14:39,787 INFO [regionserver/e611192d6313:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T22:14:39,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39087-0x101409049d30000, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-15T22:14:39,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43349-0x101409049d30003, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/e611192d6313,43349,1731708869385 2024-11-15T22:14:39,829 INFO [RS:2;e611192d6313:43349 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-15T22:14:39,853 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [e611192d6313,43349,1731708869385] 2024-11-15T22:14:39,873 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/e611192d6313,43349,1731708869385 already deleted, retry=false 2024-11-15T22:14:39,874 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of e611192d6313,43349,1731708869385 on e611192d6313,39087,1731708867142 2024-11-15T22:14:39,878 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true 2024-11-15T22:14:39,883 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=60, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true 2024-11-15T22:14:39,883 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1991): Scheduled ServerCrashProcedure pid=60 for e611192d6313,43349,1731708869385 (carryingMeta=true) e611192d6313,43349,1731708869385/CRASHED/regionCount=9/lock=java.util.concurrent.locks.ReentrantReadWriteLock@4df70035[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-15T22:14:39,884 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=60, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true, isMeta: true 2024-11-15T22:14:39,886 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/WALs/e611192d6313,43349,1731708869385-splitting 2024-11-15T22:14:39,887 INFO [PEWorker-1 {}] master.SplitWALManager(105): e611192d6313,43349,1731708869385 WAL count=1, meta=true 2024-11-15T22:14:39,889 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta}] 2024-11-15T22:14:39,894 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:39,896 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041}] 2024-11-15T22:14:39,924 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-15T22:14:39,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43349-0x101409049d30003, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T22:14:39,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43349-0x101409049d30003, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T22:14:39,953 INFO [RS:2;e611192d6313:43349 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-15T22:14:39,953 INFO [RS:2;e611192d6313:43349 {}] regionserver.HRegionServer(1031): Exiting; stopping=e611192d6313,43349,1731708869385; zookeeper connection closed. 2024-11-15T22:14:39,954 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@36d5f61e {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@36d5f61e 2024-11-15T22:14:40,058 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41231 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=62 2024-11-15T22:14:40,082 INFO [RS_LOG_REPLAY_OPS-regionserver/e611192d6313:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(299): Splitting hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/WALs/e611192d6313,43349,1731708869385-splitting/e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, size=33.0 K (33749bytes) 2024-11-15T22:14:40,082 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e611192d6313:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-15T22:14:40,083 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e611192d6313:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-15T22:14:40,083 INFO [RS_LOG_REPLAY_OPS-regionserver/e611192d6313:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/WALs/e611192d6313,43349,1731708869385-splitting/e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta 2024-11-15T22:14:40,087 INFO [RS_LOG_REPLAY_OPS-regionserver/e611192d6313:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/WALs/e611192d6313,43349,1731708869385-splitting/e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta after 3ms 2024-11-15T22:14:40,094 INFO [RS_LOG_REPLAY_OPS-regionserver/e611192d6313:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(310): Open hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/WALs/e611192d6313,43349,1731708869385-splitting/e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta took 13ms 2024-11-15T22:14:40,103 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e611192d6313:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-15T22:14:40,108 INFO [RS_LOG_REPLAY_OPS-regionserver/e611192d6313:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta.temp 2024-11-15T22:14:40,111 INFO [RS_LOG_REPLAY_OPS-regionserver/e611192d6313:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta.temp 2024-11-15T22:14:40,117 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e611192d6313:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/WALs/e611192d6313,43349,1731708869385-splitting/e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta so closing down 2024-11-15T22:14:40,117 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e611192d6313:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-15T22:14:40,139 INFO [RS_LOG_REPLAY_OPS-regionserver/e611192d6313:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.OutputSink(145): 3 split writer threads finished 2024-11-15T22:14:40,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33631 is added to blk_1073741864_1040 (size=33508) 2024-11-15T22:14:40,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741864_1040 (size=33508) 2024-11-15T22:14:40,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39269 is added to blk_1073741864_1040 (size=33508) 2024-11-15T22:14:40,150 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta.temp (wrote 61 edits, skipped 0 edits in 2 ms) 2024-11-15T22:14:40,152 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta.temp to hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-15T22:14:40,156 INFO [RS_LOG_REPLAY_OPS-regionserver/e611192d6313:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(425): Processed 62 edits across 1 Regions in 57 ms; skipped=1; WAL=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/WALs/e611192d6313,43349,1731708869385-splitting/e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, size=33.0 K, length=33749, corrupted=false, cancelled=false 2024-11-15T22:14:40,156 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e611192d6313:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(428): Completed split of hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/WALs/e611192d6313,43349,1731708869385-splitting/e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, journal: Splitting hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/WALs/e611192d6313,43349,1731708869385-splitting/e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, size=33.0 K (33749bytes) at 1731708880082Creating recovered edits writer path=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta.temp at 1731708880111 (+29 ms)Finishing writing output for hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/WALs/e611192d6313,43349,1731708869385-splitting/e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta so closing down at 1731708880117 (+6 ms)3 split writer threads finished at 1731708880139 (+22 ms)Closed recovered edits writer path=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta.temp (wrote 61 edits, skipped 0 edits in 2 ms) at 1731708880150 (+11 ms)Rename recovered edits hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta.temp to hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 at 1731708880152 (+2 ms)Processed 62 edits across 1 Regions in 57 ms; skipped=1; WAL=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/WALs/e611192d6313,43349,1731708869385-splitting/e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, size=33.0 K, length=33749, corrupted=false, cancelled=false at 1731708880156 (+4 ms) 2024-11-15T22:14:40,157 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e611192d6313:0-0 {event_type=RS_LOG_REPLAY, pid=62}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/WALs/e611192d6313,43349,1731708869385-splitting/e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta 2024-11-15T22:14:40,158 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e611192d6313:0-0 {event_type=RS_LOG_REPLAY, pid=62}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=62 2024-11-15T22:14:40,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39087 {}] master.HMaster(4169): Remote procedure done, pid=62 2024-11-15T22:14:40,166 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/WALs/e611192d6313,43349,1731708869385-splitting/e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta to hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/oldWALs 2024-11-15T22:14:40,169 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-15T22:14:40,169 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 in 270 msec 2024-11-15T22:14:40,170 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:40,173 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-15T22:14:40,174 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 in 283 msec 2024-11-15T22:14:40,176 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/WALs/e611192d6313,43349,1731708869385-splitting dir is empty, no logs to split. 2024-11-15T22:14:40,176 INFO [PEWorker-1 {}] master.SplitWALManager(105): e611192d6313,43349,1731708869385 WAL count=0, meta=true 2024-11-15T22:14:40,176 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if e611192d6313,43349,1731708869385 WAL splitting is done? wals=0, meta=true 2024-11-15T22:14:40,182 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-15T22:14:40,183 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T22:14:40,184 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-15T22:14:40,335 DEBUG [e611192d6313:39087 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-15T22:14:40,335 DEBUG [e611192d6313:39087 {}] balancer.BalancerClusterState(204): Hosts are {e611192d6313=0} racks are {/default-rack=0} 2024-11-15T22:14:40,335 DEBUG [e611192d6313:39087 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-15T22:14:40,335 DEBUG [e611192d6313:39087 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-15T22:14:40,335 DEBUG [e611192d6313:39087 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-15T22:14:40,335 DEBUG [e611192d6313:39087 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-15T22:14:40,335 INFO [e611192d6313:39087 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-15T22:14:40,335 INFO [e611192d6313:39087 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-15T22:14:40,336 DEBUG [e611192d6313:39087 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-15T22:14:40,336 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=63 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=e611192d6313,38729,1731708868111 2024-11-15T22:14:40,337 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e611192d6313,38729,1731708868111, state=OPENING 2024-11-15T22:14:40,410 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41231-0x101409049d30002, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:40,410 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39087-0x101409049d30000, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:40,410 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x101409049d30001, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:40,410 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T22:14:40,410 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:40,410 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:40,410 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:40,410 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=e611192d6313,38729,1731708868111}] 2024-11-15T22:14:40,564 INFO [regionserver/e611192d6313:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-15T22:14:40,564 INFO [regionserver/e611192d6313:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-15T22:14:40,569 INFO [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=64}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-15T22:14:40,569 INFO [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T22:14:40,569 INFO [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=64}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-15T22:14:40,571 INFO [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e611192d6313%2C38729%2C1731708868111.meta, suffix=.meta, logDir=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/WALs/e611192d6313,38729,1731708868111, archiveDir=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/oldWALs, maxLogs=32 2024-11-15T22:14:40,591 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=64}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/WALs/e611192d6313,38729,1731708868111/e611192d6313%2C38729%2C1731708868111.meta.1731708880572.meta, exclude list is [], retry=0 2024-11-15T22:14:40,595 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40641,DS-0e1f6fcf-e559-4b2f-8ba2-2f7f76775d19,DISK] 2024-11-15T22:14:40,595 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33631,DS-46524d7c-f751-41cf-9f4a-9f083a5e2adc,DISK] 2024-11-15T22:14:40,595 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39269,DS-b38187b6-2a6b-449a-a41a-28ace83252fc,DISK] 2024-11-15T22:14:40,598 INFO [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/WALs/e611192d6313,38729,1731708868111/e611192d6313%2C38729%2C1731708868111.meta.1731708880572.meta 2024-11-15T22:14:40,598 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41859:41859),(127.0.0.1/127.0.0.1:34309:34309),(127.0.0.1/127.0.0.1:45219:45219)] 2024-11-15T22:14:40,598 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-15T22:14:40,599 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=64}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-15T22:14:40,599 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-15T22:14:40,599 INFO [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-15T22:14:40,599 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-15T22:14:40,599 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:40,599 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-15T22:14:40,600 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-15T22:14:40,602 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-15T22:14:40,603 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-15T22:14:40,604 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:40,604 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T22:14:40,604 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-15T22:14:40,606 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-15T22:14:40,606 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:40,606 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T22:14:40,606 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-15T22:14:40,608 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-15T22:14:40,608 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:40,608 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T22:14:40,608 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-15T22:14:40,609 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-15T22:14:40,609 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:40,610 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T22:14:40,610 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-15T22:14:40,611 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/hbase/meta/1588230740 2024-11-15T22:14:40,613 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/hbase/meta/1588230740 2024-11-15T22:14:40,614 INFO [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-15T22:14:40,640 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5793): Applied 277, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=64, path=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-15T22:14:40,643 INFO [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=29.41 KB heapSize=50.67 KB 2024-11-15T22:14:40,715 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/hbase/meta/1588230740/.tmp/info/5bc5921e08444a1eac90f9058697c2d6 is 138, key is test,bbb,1731708873681.897a7bb57a002d365489ce583eb7f84f./info:regioninfo/1731708875378/Put/seqid=0 2024-11-15T22:14:40,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741866_1042 (size=36385) 2024-11-15T22:14:40,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39269 is added to blk_1073741866_1042 (size=36385) 2024-11-15T22:14:40,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33631 is added to blk_1073741866_1042 (size=36385) 2024-11-15T22:14:40,727 INFO [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.26 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/hbase/meta/1588230740/.tmp/info/5bc5921e08444a1eac90f9058697c2d6 2024-11-15T22:14:40,784 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/hbase/meta/1588230740/.tmp/ns/6494057806b74e179cdae59270402ef7 is 43, key is default/ns:d/1731708873428/Put/seqid=0 2024-11-15T22:14:40,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39269 is added to blk_1073741867_1043 (size=5153) 2024-11-15T22:14:40,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741867_1043 (size=5153) 2024-11-15T22:14:40,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33631 is added to blk_1073741867_1043 (size=5153) 2024-11-15T22:14:40,793 INFO [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/hbase/meta/1588230740/.tmp/ns/6494057806b74e179cdae59270402ef7 2024-11-15T22:14:40,816 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/hbase/meta/1588230740/.tmp/table/1ca4f647a5b145feae8839ed588f151e is 40, key is test/table:state/1731708876063/Put/seqid=0 2024-11-15T22:14:40,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39269 is added to blk_1073741868_1044 (size=5165) 2024-11-15T22:14:40,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33631 is added to blk_1073741868_1044 (size=5165) 2024-11-15T22:14:40,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741868_1044 (size=5165) 2024-11-15T22:14:40,824 INFO [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=72 B at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/hbase/meta/1588230740/.tmp/table/1ca4f647a5b145feae8839ed588f151e 2024-11-15T22:14:40,832 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/hbase/meta/1588230740/.tmp/info/5bc5921e08444a1eac90f9058697c2d6 as hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/hbase/meta/1588230740/info/5bc5921e08444a1eac90f9058697c2d6 2024-11-15T22:14:40,840 INFO [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/hbase/meta/1588230740/info/5bc5921e08444a1eac90f9058697c2d6, entries=273, sequenceid=64, filesize=35.5 K 2024-11-15T22:14:40,842 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/hbase/meta/1588230740/.tmp/ns/6494057806b74e179cdae59270402ef7 as hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/hbase/meta/1588230740/ns/6494057806b74e179cdae59270402ef7 2024-11-15T22:14:40,850 INFO [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/hbase/meta/1588230740/ns/6494057806b74e179cdae59270402ef7, entries=2, sequenceid=64, filesize=5.0 K 2024-11-15T22:14:40,852 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/hbase/meta/1588230740/.tmp/table/1ca4f647a5b145feae8839ed588f151e as hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/hbase/meta/1588230740/table/1ca4f647a5b145feae8839ed588f151e 2024-11-15T22:14:40,859 INFO [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/hbase/meta/1588230740/table/1ca4f647a5b145feae8839ed588f151e, entries=2, sequenceid=64, filesize=5.0 K 2024-11-15T22:14:40,862 INFO [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(3140): Finished flush of dataSize ~29.41 KB/30111, heapSize ~50.38 KB/51584, currentSize=0 B/0 for 1588230740 in 219ms, sequenceid=64, compaction requested=false; wal=null 2024-11-15T22:14:40,864 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-15T22:14:40,865 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-15T22:14:40,865 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-15T22:14:40,866 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-15T22:14:40,867 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-15T22:14:40,870 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/hbase/meta/1588230740/recovered.edits/64.seqid, newMaxSeqId=64, maxSeqId=1 2024-11-15T22:14:40,871 INFO [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=65; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68082952, jitterRate=0.014515042304992676}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-15T22:14:40,871 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-15T22:14:40,871 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731708880600Writing region info on filesystem at 1731708880600Initializing all the Stores at 1731708880601 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731708880601Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731708880602 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708880602Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731708880602Obtaining lock to block concurrent updates at 1731708880643 (+41 ms)Preparing flush snapshotting stores in 1588230740 at 1731708880643Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=30111, getHeapSize=51824, getOffHeapSize=0, getCellsCount=277 at 1731708880646 (+3 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731708880646Flushing 1588230740/info: creating writer at 1731708880647 (+1 ms)Flushing 1588230740/info: appending metadata at 1731708880705 (+58 ms)Flushing 1588230740/info: closing flushed file at 1731708880708 (+3 ms)Flushing 1588230740/ns: creating writer at 1731708880768 (+60 ms)Flushing 1588230740/ns: appending metadata at 1731708880783 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1731708880783Flushing 1588230740/table: creating writer at 1731708880801 (+18 ms)Flushing 1588230740/table: appending metadata at 1731708880815 (+14 ms)Flushing 1588230740/table: closing flushed file at 1731708880815Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@185ffc7c: reopening flushed file at 1731708880831 (+16 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@27333366: reopening flushed file at 1731708880841 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3274486d: reopening flushed file at 1731708880851 (+10 ms)Finished flush of dataSize ~29.41 KB/30111, heapSize ~50.38 KB/51584, currentSize=0 B/0 for 1588230740 in 219ms, sequenceid=64, compaction requested=false; wal=null at 1731708880862 (+11 ms)Cleaning up temporary data from old regions at 1731708880865 (+3 ms)Running coprocessor post-open hooks at 1731708880871 (+6 ms)Region opened successfully at 1731708880871 2024-11-15T22:14:40,873 INFO [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=64, masterSystemTime=1731708880563 2024-11-15T22:14:40,875 DEBUG [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-15T22:14:40,875 INFO [RS_OPEN_META-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_META, pid=64}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-15T22:14:40,876 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=63 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=65, regionLocation=e611192d6313,38729,1731708868111 2024-11-15T22:14:40,877 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e611192d6313,38729,1731708868111, state=OPEN 2024-11-15T22:14:40,926 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39087-0x101409049d30000, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:40,926 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41231-0x101409049d30002, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:40,926 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x101409049d30001, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:40,926 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=64, ppid=63, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=e611192d6313,38729,1731708868111 2024-11-15T22:14:40,926 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:40,926 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:40,926 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:40,929 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-15T22:14:40,929 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e611192d6313,38729,1731708868111 in 516 msec 2024-11-15T22:14:40,931 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=60 2024-11-15T22:14:40,932 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 747 msec 2024-11-15T22:14:40,932 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): e611192d6313,43349,1731708869385 had 9 regions 2024-11-15T22:14:40,933 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=60, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true, isMeta: false 2024-11-15T22:14:40,935 INFO [PEWorker-1 {}] master.SplitWALManager(105): e611192d6313,43349,1731708869385 WAL count=1, meta=false 2024-11-15T22:14:40,936 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=60, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.1731708872733}] 2024-11-15T22:14:40,938 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:40,939 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041}] 2024-11-15T22:14:41,092 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41231 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=66 2024-11-15T22:14:41,107 INFO [RS_LOG_REPLAY_OPS-regionserver/e611192d6313:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(299): Splitting hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/WALs/e611192d6313,43349,1731708869385-splitting/e611192d6313%2C43349%2C1731708869385.1731708872733, size=2.7 K (2769bytes) 2024-11-15T22:14:41,107 INFO [RS_LOG_REPLAY_OPS-regionserver/e611192d6313:0-1 {event_type=RS_LOG_REPLAY, pid=66}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/WALs/e611192d6313,43349,1731708869385-splitting/e611192d6313%2C43349%2C1731708869385.1731708872733 2024-11-15T22:14:41,107 INFO [RS_LOG_REPLAY_OPS-regionserver/e611192d6313:0-1 {event_type=RS_LOG_REPLAY, pid=66}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/WALs/e611192d6313,43349,1731708869385-splitting/e611192d6313%2C43349%2C1731708869385.1731708872733 after 0ms 2024-11-15T22:14:41,110 INFO [RS_LOG_REPLAY_OPS-regionserver/e611192d6313:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(310): Open hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/WALs/e611192d6313,43349,1731708869385-splitting/e611192d6313%2C43349%2C1731708869385.1731708872733 took 4ms 2024-11-15T22:14:41,113 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e611192d6313:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 2e79a8ce6f9715045d056d3dfdef5cec: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-15T22:14:41,115 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e611192d6313:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 06b9df8f55d791077e9f7ec55ef6a971: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-15T22:14:41,117 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e611192d6313:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 93deaad3e3229244196e8daa9478371a: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-15T22:14:41,118 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e611192d6313:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 2a159d388ea2138ee8e2bed061a785ce: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-15T22:14:41,120 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e611192d6313:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 13c3457b9645c9a8181ed9b4551962ad: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-15T22:14:41,121 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e611192d6313:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for f2783381a48c13c2f75c07e5029903cb: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-15T22:14:41,123 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e611192d6313:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 897a7bb57a002d365489ce583eb7f84f: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-15T22:14:41,124 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e611192d6313:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for c56d152c07a95a189ac65341ff4db1fa: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-15T22:14:41,126 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e611192d6313:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 819421a6406c0ebb5635c399b2c0303d: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-15T22:14:41,126 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e611192d6313:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/WALs/e611192d6313,43349,1731708869385-splitting/e611192d6313%2C43349%2C1731708869385.1731708872733 so closing down 2024-11-15T22:14:41,127 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e611192d6313:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-15T22:14:41,127 INFO [RS_LOG_REPLAY_OPS-regionserver/e611192d6313:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.OutputSink(145): 3 split writer threads finished 2024-11-15T22:14:41,127 INFO [RS_LOG_REPLAY_OPS-regionserver/e611192d6313:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(425): Processed 10 edits across 0 Regions in 16 ms; skipped=10; WAL=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/WALs/e611192d6313,43349,1731708869385-splitting/e611192d6313%2C43349%2C1731708869385.1731708872733, size=2.7 K, length=2769, corrupted=false, cancelled=false 2024-11-15T22:14:41,127 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e611192d6313:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(428): Completed split of hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/WALs/e611192d6313,43349,1731708869385-splitting/e611192d6313%2C43349%2C1731708869385.1731708872733, journal: Splitting hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/WALs/e611192d6313,43349,1731708869385-splitting/e611192d6313%2C43349%2C1731708869385.1731708872733, size=2.7 K (2769bytes) at 1731708881107Finishing writing output for hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/WALs/e611192d6313,43349,1731708869385-splitting/e611192d6313%2C43349%2C1731708869385.1731708872733 so closing down at 1731708881126 (+19 ms)3 split writer threads finished at 1731708881127 (+1 ms)Processed 10 edits across 0 Regions in 16 ms; skipped=10; WAL=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/WALs/e611192d6313,43349,1731708869385-splitting/e611192d6313%2C43349%2C1731708869385.1731708872733, size=2.7 K, length=2769, corrupted=false, cancelled=false at 1731708881127 2024-11-15T22:14:41,127 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e611192d6313:0-1 {event_type=RS_LOG_REPLAY, pid=66}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/WALs/e611192d6313,43349,1731708869385-splitting/e611192d6313%2C43349%2C1731708869385.1731708872733 2024-11-15T22:14:41,127 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e611192d6313:0-1 {event_type=RS_LOG_REPLAY, pid=66}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=66 2024-11-15T22:14:41,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39087 {}] master.HMaster(4169): Remote procedure done, pid=66 2024-11-15T22:14:41,131 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/WALs/e611192d6313,43349,1731708869385-splitting/e611192d6313%2C43349%2C1731708869385.1731708872733 to hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/oldWALs 2024-11-15T22:14:41,133 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=66, resume processing ppid=65 2024-11-15T22:14:41,133 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 in 192 msec 2024-11-15T22:14:41,135 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:41,137 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=60 2024-11-15T22:14:41,137 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 in 200 msec 2024-11-15T22:14:41,139 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/WALs/e611192d6313,43349,1731708869385-splitting dir is empty, no logs to split. 2024-11-15T22:14:41,139 INFO [PEWorker-1 {}] master.SplitWALManager(105): e611192d6313,43349,1731708869385 WAL count=0, meta=false 2024-11-15T22:14:41,139 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if e611192d6313,43349,1731708869385 WAL splitting is done? wals=0, meta=false 2024-11-15T22:14:41,141 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN}, {pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN}, {pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN}, {pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f2783381a48c13c2f75c07e5029903cb, ASSIGN}, {pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=06b9df8f55d791077e9f7ec55ef6a971, ASSIGN}, {pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=897a7bb57a002d365489ce583eb7f84f, ASSIGN}, {pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c56d152c07a95a189ac65341ff4db1fa, ASSIGN}, {pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2e79a8ce6f9715045d056d3dfdef5cec, ASSIGN}] 2024-11-15T22:14:41,142 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2e79a8ce6f9715045d056d3dfdef5cec, ASSIGN 2024-11-15T22:14:41,143 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=2e79a8ce6f9715045d056d3dfdef5cec, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-15T22:14:41,144 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c56d152c07a95a189ac65341ff4db1fa, ASSIGN 2024-11-15T22:14:41,145 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c56d152c07a95a189ac65341ff4db1fa, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-15T22:14:41,145 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=897a7bb57a002d365489ce583eb7f84f, ASSIGN 2024-11-15T22:14:41,146 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=897a7bb57a002d365489ce583eb7f84f, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-15T22:14:41,147 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=06b9df8f55d791077e9f7ec55ef6a971, ASSIGN 2024-11-15T22:14:41,148 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=06b9df8f55d791077e9f7ec55ef6a971, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-15T22:14:41,149 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f2783381a48c13c2f75c07e5029903cb, ASSIGN 2024-11-15T22:14:41,149 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f2783381a48c13c2f75c07e5029903cb, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-15T22:14:41,150 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN 2024-11-15T22:14:41,151 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-15T22:14:41,151 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN 2024-11-15T22:14:41,152 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-15T22:14:41,153 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN 2024-11-15T22:14:41,154 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-15T22:14:41,294 DEBUG [e611192d6313:39087 {}] balancer.BalancerClusterState(204): Hosts are {e611192d6313=0} racks are {/default-rack=0} 2024-11-15T22:14:41,294 DEBUG [e611192d6313:39087 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-15T22:14:41,294 DEBUG [e611192d6313:39087 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-15T22:14:41,294 DEBUG [e611192d6313:39087 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-15T22:14:41,294 DEBUG [e611192d6313:39087 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-15T22:14:41,294 INFO [e611192d6313:39087 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-15T22:14:41,294 INFO [e611192d6313:39087 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-15T22:14:41,294 DEBUG [e611192d6313:39087 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-15T22:14:41,295 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=74 updating hbase:meta row=2e79a8ce6f9715045d056d3dfdef5cec, regionState=OPENING, regionLocation=e611192d6313,41231,1731708869041 2024-11-15T22:14:41,296 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=06b9df8f55d791077e9f7ec55ef6a971, regionState=OPENING, regionLocation=e611192d6313,41231,1731708869041 2024-11-15T22:14:41,296 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=2a159d388ea2138ee8e2bed061a785ce, regionState=OPENING, regionLocation=e611192d6313,41231,1731708869041 2024-11-15T22:14:41,296 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server e611192d6313:43349 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: e611192d6313/172.17.0.3:43349 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:41,297 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=72 updating hbase:meta row=897a7bb57a002d365489ce583eb7f84f, regionState=OPENING, regionLocation=e611192d6313,41231,1731708869041 2024-11-15T22:14:41,298 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=819421a6406c0ebb5635c399b2c0303d, regionState=OPENING, regionLocation=e611192d6313,38729,1731708868111 2024-11-15T22:14:41,298 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=e611192d6313,43349,1731708869385, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=e611192d6313,43349,1731708869385, seqNum=-1, error=java.net.ConnectException: Call to address=e611192d6313:43349 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: e611192d6313/172.17.0.3:43349 2024-11-15T22:14:41,299 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=73 updating hbase:meta row=c56d152c07a95a189ac65341ff4db1fa, regionState=OPENING, regionLocation=e611192d6313,38729,1731708868111 2024-11-15T22:14:41,299 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=e611192d6313,43349,1731708869385, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-15T22:14:41,299 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=e611192d6313,43349,1731708869385, seqNum=-1 from cache 2024-11-15T22:14:41,299 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=13c3457b9645c9a8181ed9b4551962ad, regionState=OPENING, regionLocation=e611192d6313,38729,1731708868111 2024-11-15T22:14:41,299 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T22:14:41,300 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e611192d6313,38729,1731708868111, seqNum=-1] 2024-11-15T22:14:41,300 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address e611192d6313:43349 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: e611192d6313/172.17.0.3:43349 2024-11-15T22:14:41,300 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T22:14:41,301 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=f2783381a48c13c2f75c07e5029903cb, regionState=OPENING, regionLocation=e611192d6313,38729,1731708868111 2024-11-15T22:14:41,301 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server e611192d6313:43349 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: e611192d6313/172.17.0.3:43349 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:41,301 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=e611192d6313,43349,1731708869385, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=e611192d6313,38729,1731708868111, seqNum=-1, error=java.net.ConnectException: Call to address=e611192d6313:43349 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: e611192d6313/172.17.0.3:43349 2024-11-15T22:14:41,301 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=e611192d6313,43349,1731708869385, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=e611192d6313,38729,1731708868111, seqNum=-1, error=java.net.ConnectException: Call to address=e611192d6313:43349 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: e611192d6313/172.17.0.3:43349 2024-11-15T22:14:41,301 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=e611192d6313,43349,1731708869385, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=e611192d6313,38729,1731708868111, seqNum=-1, error=java.net.ConnectException: Call to address=e611192d6313:43349 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: e611192d6313/172.17.0.3:43349 2024-11-15T22:14:41,302 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=e611192d6313,43349,1731708869385, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=e611192d6313,38729,1731708868111, seqNum=-1, error=java.net.ConnectException: Call to address=e611192d6313:43349 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: e611192d6313/172.17.0.3:43349 2024-11-15T22:14:41,302 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57165, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T22:14:41,302 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=e611192d6313,43349,1731708869385, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=e611192d6313,38729,1731708868111, seqNum=-1, error=java.net.ConnectException: Call to address=e611192d6313:43349 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: e611192d6313/172.17.0.3:43349 2024-11-15T22:14:41,302 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address e611192d6313:43349 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: e611192d6313/172.17.0.3:43349 2024-11-15T22:14:41,304 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN because future has completed 2024-11-15T22:14:41,304 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111}] 2024-11-15T22:14:41,305 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f2783381a48c13c2f75c07e5029903cb, ASSIGN because future has completed 2024-11-15T22:14:41,308 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure f2783381a48c13c2f75c07e5029903cb, server=e611192d6313,38729,1731708868111}] 2024-11-15T22:14:41,409 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=2e79a8ce6f9715045d056d3dfdef5cec, ASSIGN because future has completed 2024-11-15T22:14:41,410 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2e79a8ce6f9715045d056d3dfdef5cec, server=e611192d6313,41231,1731708869041}] 2024-11-15T22:14:41,411 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=897a7bb57a002d365489ce583eb7f84f, ASSIGN because future has completed 2024-11-15T22:14:41,412 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN because future has completed 2024-11-15T22:14:41,413 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c56d152c07a95a189ac65341ff4db1fa, ASSIGN because future has completed 2024-11-15T22:14:41,414 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure c56d152c07a95a189ac65341ff4db1fa, server=e611192d6313,38729,1731708868111}] 2024-11-15T22:14:41,414 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=06b9df8f55d791077e9f7ec55ef6a971, ASSIGN because future has completed 2024-11-15T22:14:41,415 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN because future has completed 2024-11-15T22:14:41,417 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2a159d388ea2138ee8e2bed061a785ce, server=e611192d6313,41231,1731708869041}] 2024-11-15T22:14:41,419 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06b9df8f55d791077e9f7ec55ef6a971, server=e611192d6313,41231,1731708869041}] 2024-11-15T22:14:41,422 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 819421a6406c0ebb5635c399b2c0303d, server=e611192d6313,38729,1731708868111}] 2024-11-15T22:14:41,424 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 897a7bb57a002d365489ce583eb7f84f, server=e611192d6313,41231,1731708869041}] 2024-11-15T22:14:41,461 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(132): Open test,hhh,1731708873681.13c3457b9645c9a8181ed9b4551962ad. 2024-11-15T22:14:41,461 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7752): Opening region: {ENCODED => 13c3457b9645c9a8181ed9b4551962ad, NAME => 'test,hhh,1731708873681.13c3457b9645c9a8181ed9b4551962ad.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-15T22:14:41,462 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 13c3457b9645c9a8181ed9b4551962ad 2024-11-15T22:14:41,462 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(898): Instantiated test,hhh,1731708873681.13c3457b9645c9a8181ed9b4551962ad.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:41,462 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7794): checking encryption for 13c3457b9645c9a8181ed9b4551962ad 2024-11-15T22:14:41,462 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7797): checking classloading for 13c3457b9645c9a8181ed9b4551962ad 2024-11-15T22:14:41,464 INFO [StoreOpener-13c3457b9645c9a8181ed9b4551962ad-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 13c3457b9645c9a8181ed9b4551962ad 2024-11-15T22:14:41,465 INFO [StoreOpener-13c3457b9645c9a8181ed9b4551962ad-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 13c3457b9645c9a8181ed9b4551962ad columnFamilyName family 2024-11-15T22:14:41,465 DEBUG [StoreOpener-13c3457b9645c9a8181ed9b4551962ad-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:41,465 INFO [StoreOpener-13c3457b9645c9a8181ed9b4551962ad-1 {}] regionserver.HStore(327): Store=13c3457b9645c9a8181ed9b4551962ad/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:14:41,465 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1038): replaying wal for 13c3457b9645c9a8181ed9b4551962ad 2024-11-15T22:14:41,466 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/13c3457b9645c9a8181ed9b4551962ad 2024-11-15T22:14:41,468 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/13c3457b9645c9a8181ed9b4551962ad 2024-11-15T22:14:41,468 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1048): stopping wal replay for 13c3457b9645c9a8181ed9b4551962ad 2024-11-15T22:14:41,468 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1060): Cleaning up temporary data for 13c3457b9645c9a8181ed9b4551962ad 2024-11-15T22:14:41,470 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): writing seq id for 13c3457b9645c9a8181ed9b4551962ad 2024-11-15T22:14:41,471 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1114): Opened 13c3457b9645c9a8181ed9b4551962ad; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62089179, jitterRate=-0.07479913532733917}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T22:14:41,471 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 13c3457b9645c9a8181ed9b4551962ad 2024-11-15T22:14:41,471 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1006): Region open journal for 13c3457b9645c9a8181ed9b4551962ad: Running coprocessor pre-open hook at 1731708881462Writing region info on filesystem at 1731708881462Initializing all the Stores at 1731708881463 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708881463Cleaning up temporary data from old regions at 1731708881468 (+5 ms)Running coprocessor post-open hooks at 1731708881471 (+3 ms)Region opened successfully at 1731708881471 2024-11-15T22:14:41,472 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1731708873681.13c3457b9645c9a8181ed9b4551962ad., pid=75, masterSystemTime=1731708881457 2024-11-15T22:14:41,474 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1731708873681.13c3457b9645c9a8181ed9b4551962ad. 2024-11-15T22:14:41,474 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(153): Opened test,hhh,1731708873681.13c3457b9645c9a8181ed9b4551962ad. 2024-11-15T22:14:41,474 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(132): Open test,nnn,1731708873681.c56d152c07a95a189ac65341ff4db1fa. 2024-11-15T22:14:41,474 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7752): Opening region: {ENCODED => c56d152c07a95a189ac65341ff4db1fa, NAME => 'test,nnn,1731708873681.c56d152c07a95a189ac65341ff4db1fa.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-15T22:14:41,474 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c56d152c07a95a189ac65341ff4db1fa 2024-11-15T22:14:41,475 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(898): Instantiated test,nnn,1731708873681.c56d152c07a95a189ac65341ff4db1fa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:41,475 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7794): checking encryption for c56d152c07a95a189ac65341ff4db1fa 2024-11-15T22:14:41,475 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7797): checking classloading for c56d152c07a95a189ac65341ff4db1fa 2024-11-15T22:14:41,476 WARN [PEWorker-1 {}] procedure2.ProcedureTestingUtility(201): Set Kill before store update in rollback to: true 2024-11-15T22:14:41,476 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111 2024-11-15T22:14:41,476 INFO [StoreOpener-c56d152c07a95a189ac65341ff4db1fa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c56d152c07a95a189ac65341ff4db1fa 2024-11-15T22:14:41,476 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1844): CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111 java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:14:41,478 INFO [StoreOpener-c56d152c07a95a189ac65341ff4db1fa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c56d152c07a95a189ac65341ff4db1fa columnFamilyName family 2024-11-15T22:14:41,478 DEBUG [StoreOpener-c56d152c07a95a189ac65341ff4db1fa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:41,478 INFO [StoreOpener-c56d152c07a95a189ac65341ff4db1fa-1 {}] regionserver.HStore(327): Store=c56d152c07a95a189ac65341ff4db1fa/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:14:41,479 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1038): replaying wal for c56d152c07a95a189ac65341ff4db1fa 2024-11-15T22:14:41,479 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/c56d152c07a95a189ac65341ff4db1fa 2024-11-15T22:14:41,481 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/c56d152c07a95a189ac65341ff4db1fa 2024-11-15T22:14:41,482 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1048): stopping wal replay for c56d152c07a95a189ac65341ff4db1fa 2024-11-15T22:14:41,482 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1060): Cleaning up temporary data for c56d152c07a95a189ac65341ff4db1fa 2024-11-15T22:14:41,484 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1093): writing seq id for c56d152c07a95a189ac65341ff4db1fa 2024-11-15T22:14:41,485 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1114): Opened c56d152c07a95a189ac65341ff4db1fa; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71001474, jitterRate=0.058004409074783325}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T22:14:41,486 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c56d152c07a95a189ac65341ff4db1fa 2024-11-15T22:14:41,486 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1006): Region open journal for c56d152c07a95a189ac65341ff4db1fa: Running coprocessor pre-open hook at 1731708881475Writing region info on filesystem at 1731708881475Initializing all the Stores at 1731708881476 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708881476Cleaning up temporary data from old regions at 1731708881482 (+6 ms)Running coprocessor post-open hooks at 1731708881486 (+4 ms)Region opened successfully at 1731708881486 2024-11-15T22:14:41,487 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2236): Post open deploy tasks for test,nnn,1731708873681.c56d152c07a95a189ac65341ff4db1fa., pid=78, masterSystemTime=1731708881457 2024-11-15T22:14:41,486 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=true, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1845) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 4 more 2024-11-15T22:14:41,487 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-15T22:14:41,487 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=82, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 897a7bb57a002d365489ce583eb7f84f, server=e611192d6313,41231,1731708869041 2024-11-15T22:14:41,487 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T22:14:41,488 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=82, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 897a7bb57a002d365489ce583eb7f84f, server=e611192d6313,41231,1731708869041 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:14:41,489 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2266): Finished post open deploy task for test,nnn,1731708873681.c56d152c07a95a189ac65341ff4db1fa. 2024-11-15T22:14:41,489 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(153): Opened test,nnn,1731708873681.c56d152c07a95a189ac65341ff4db1fa. 2024-11-15T22:14:41,489 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(132): Open test,sss,1731708873681.819421a6406c0ebb5635c399b2c0303d. 2024-11-15T22:14:41,490 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7752): Opening region: {ENCODED => 819421a6406c0ebb5635c399b2c0303d, NAME => 'test,sss,1731708873681.819421a6406c0ebb5635c399b2c0303d.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-15T22:14:41,490 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 819421a6406c0ebb5635c399b2c0303d 2024-11-15T22:14:41,490 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(898): Instantiated test,sss,1731708873681.819421a6406c0ebb5635c399b2c0303d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:41,490 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7794): checking encryption for 819421a6406c0ebb5635c399b2c0303d 2024-11-15T22:14:41,490 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7797): checking classloading for 819421a6406c0ebb5635c399b2c0303d 2024-11-15T22:14:41,492 INFO [StoreOpener-819421a6406c0ebb5635c399b2c0303d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 819421a6406c0ebb5635c399b2c0303d 2024-11-15T22:14:41,493 INFO [StoreOpener-819421a6406c0ebb5635c399b2c0303d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 819421a6406c0ebb5635c399b2c0303d columnFamilyName family 2024-11-15T22:14:41,493 DEBUG [StoreOpener-819421a6406c0ebb5635c399b2c0303d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:41,493 INFO [StoreOpener-819421a6406c0ebb5635c399b2c0303d-1 {}] regionserver.HStore(327): Store=819421a6406c0ebb5635c399b2c0303d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:14:41,493 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1038): replaying wal for 819421a6406c0ebb5635c399b2c0303d 2024-11-15T22:14:41,494 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/819421a6406c0ebb5635c399b2c0303d 2024-11-15T22:14:41,496 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/819421a6406c0ebb5635c399b2c0303d 2024-11-15T22:14:41,496 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1048): stopping wal replay for 819421a6406c0ebb5635c399b2c0303d 2024-11-15T22:14:41,496 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1060): Cleaning up temporary data for 819421a6406c0ebb5635c399b2c0303d 2024-11-15T22:14:41,504 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1093): writing seq id for 819421a6406c0ebb5635c399b2c0303d 2024-11-15T22:14:41,505 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1114): Opened 819421a6406c0ebb5635c399b2c0303d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71256887, jitterRate=0.06181035935878754}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T22:14:41,506 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 819421a6406c0ebb5635c399b2c0303d 2024-11-15T22:14:41,506 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1006): Region open journal for 819421a6406c0ebb5635c399b2c0303d: Running coprocessor pre-open hook at 1731708881490Writing region info on filesystem at 1731708881490Initializing all the Stores at 1731708881491 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708881492 (+1 ms)Cleaning up temporary data from old regions at 1731708881496 (+4 ms)Running coprocessor post-open hooks at 1731708881506 (+10 ms)Region opened successfully at 1731708881506 2024-11-15T22:14:41,506 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1731708873681.819421a6406c0ebb5635c399b2c0303d., pid=81, masterSystemTime=1731708881457 2024-11-15T22:14:41,509 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1731708873681.819421a6406c0ebb5635c399b2c0303d. 2024-11-15T22:14:41,509 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(153): Opened test,sss,1731708873681.819421a6406c0ebb5635c399b2c0303d. 2024-11-15T22:14:41,509 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(132): Open test,,1731708873681.f2783381a48c13c2f75c07e5029903cb. 2024-11-15T22:14:41,509 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7752): Opening region: {ENCODED => f2783381a48c13c2f75c07e5029903cb, NAME => 'test,,1731708873681.f2783381a48c13c2f75c07e5029903cb.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-15T22:14:41,509 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f2783381a48c13c2f75c07e5029903cb 2024-11-15T22:14:41,509 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(898): Instantiated test,,1731708873681.f2783381a48c13c2f75c07e5029903cb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:41,509 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7794): checking encryption for f2783381a48c13c2f75c07e5029903cb 2024-11-15T22:14:41,510 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7797): checking classloading for f2783381a48c13c2f75c07e5029903cb 2024-11-15T22:14:41,511 INFO [StoreOpener-f2783381a48c13c2f75c07e5029903cb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f2783381a48c13c2f75c07e5029903cb 2024-11-15T22:14:41,512 INFO [StoreOpener-f2783381a48c13c2f75c07e5029903cb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f2783381a48c13c2f75c07e5029903cb columnFamilyName family 2024-11-15T22:14:41,512 DEBUG [StoreOpener-f2783381a48c13c2f75c07e5029903cb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:41,512 INFO [StoreOpener-f2783381a48c13c2f75c07e5029903cb-1 {}] regionserver.HStore(327): Store=f2783381a48c13c2f75c07e5029903cb/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:14:41,512 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1038): replaying wal for f2783381a48c13c2f75c07e5029903cb 2024-11-15T22:14:41,513 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/f2783381a48c13c2f75c07e5029903cb 2024-11-15T22:14:41,514 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/f2783381a48c13c2f75c07e5029903cb 2024-11-15T22:14:41,515 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1048): stopping wal replay for f2783381a48c13c2f75c07e5029903cb 2024-11-15T22:14:41,515 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1060): Cleaning up temporary data for f2783381a48c13c2f75c07e5029903cb 2024-11-15T22:14:41,516 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1093): writing seq id for f2783381a48c13c2f75c07e5029903cb 2024-11-15T22:14:41,517 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1114): Opened f2783381a48c13c2f75c07e5029903cb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69703451, jitterRate=0.03866235911846161}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T22:14:41,517 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f2783381a48c13c2f75c07e5029903cb 2024-11-15T22:14:41,518 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1006): Region open journal for f2783381a48c13c2f75c07e5029903cb: Running coprocessor pre-open hook at 1731708881510Writing region info on filesystem at 1731708881510Initializing all the Stores at 1731708881510Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708881511 (+1 ms)Cleaning up temporary data from old regions at 1731708881515 (+4 ms)Running coprocessor post-open hooks at 1731708881517 (+2 ms)Region opened successfully at 1731708881518 (+1 ms) 2024-11-15T22:14:41,518 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1731708873681.f2783381a48c13c2f75c07e5029903cb., pid=76, masterSystemTime=1731708881457 2024-11-15T22:14:41,520 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2266): Finished post open deploy task for test,,1731708873681.f2783381a48c13c2f75c07e5029903cb. 2024-11-15T22:14:41,520 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(153): Opened test,,1731708873681.f2783381a48c13c2f75c07e5029903cb. 2024-11-15T22:14:41,527 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T22:14:41,527 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T22:14:41,527 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T22:14:41,527 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T22:14:41,528 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T22:14:41,528 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T22:14:41,528 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T22:14:41,528 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T22:14:41,529 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-15T22:14:41,534 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-15T22:14:41,535 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T22:14:41,535 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T22:14:41,535 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,535 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=81, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T22:14:41,535 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,535 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T22:14:41,535 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,535 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T22:14:41,535 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,535 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T22:14:41,535 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,535 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T22:14:41,535 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,535 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=82, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T22:14:41,535 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,535 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T22:14:41,535 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,535 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T22:14:41,535 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T22:14:41,535 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T22:14:41,535 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,536 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T22:14:41,536 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T22:14:41,537 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-15T22:14:41,538 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=93deaad3e3229244196e8daa9478371a, REOPEN/MOVE 2024-11-15T22:14:41,541 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true 2024-11-15T22:14:41,541 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:41,541 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:41,541 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T22:14:41,541 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e611192d6313,38729,1731708868111 2024-11-15T22:14:41,541 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:41,541 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:41,541 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN 2024-11-15T22:14:41,541 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN 2024-11-15T22:14:41,542 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN 2024-11-15T22:14:41,542 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f2783381a48c13c2f75c07e5029903cb, ASSIGN 2024-11-15T22:14:41,542 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06b9df8f55d791077e9f7ec55ef6a971, ASSIGN 2024-11-15T22:14:41,542 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=897a7bb57a002d365489ce583eb7f84f, ASSIGN 2024-11-15T22:14:41,542 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c56d152c07a95a189ac65341ff4db1fa, ASSIGN 2024-11-15T22:14:41,542 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e79a8ce6f9715045d056d3dfdef5cec, ASSIGN 2024-11-15T22:14:41,542 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111 2024-11-15T22:14:41,542 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure f2783381a48c13c2f75c07e5029903cb, server=e611192d6313,38729,1731708868111 2024-11-15T22:14:41,542 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2e79a8ce6f9715045d056d3dfdef5cec, server=e611192d6313,41231,1731708869041 2024-11-15T22:14:41,542 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure c56d152c07a95a189ac65341ff4db1fa, server=e611192d6313,38729,1731708868111 2024-11-15T22:14:41,542 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2a159d388ea2138ee8e2bed061a785ce, server=e611192d6313,41231,1731708869041 2024-11-15T22:14:41,542 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06b9df8f55d791077e9f7ec55ef6a971, server=e611192d6313,41231,1731708869041 2024-11-15T22:14:41,542 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=81, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 819421a6406c0ebb5635c399b2c0303d, server=e611192d6313,38729,1731708868111 2024-11-15T22:14:41,542 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=82, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 897a7bb57a002d365489ce583eb7f84f, server=e611192d6313,41231,1731708869041 2024-11-15T22:14:41,542 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,542 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,542 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,542 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,542 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e611192d6313,38729,1731708868111 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,542 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,542 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,542 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,542 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN 2024-11-15T22:14:41,542 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,543 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN 2024-11-15T22:14:41,543 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,543 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN 2024-11-15T22:14:41,543 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f2783381a48c13c2f75c07e5029903cb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,543 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f2783381a48c13c2f75c07e5029903cb, ASSIGN 2024-11-15T22:14:41,543 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06b9df8f55d791077e9f7ec55ef6a971, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,543 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06b9df8f55d791077e9f7ec55ef6a971, ASSIGN 2024-11-15T22:14:41,543 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=897a7bb57a002d365489ce583eb7f84f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,543 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=897a7bb57a002d365489ce583eb7f84f, ASSIGN 2024-11-15T22:14:41,543 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c56d152c07a95a189ac65341ff4db1fa, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,543 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c56d152c07a95a189ac65341ff4db1fa, ASSIGN 2024-11-15T22:14:41,543 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e79a8ce6f9715045d056d3dfdef5cec, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,543 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e79a8ce6f9715045d056d3dfdef5cec, ASSIGN 2024-11-15T22:14:41,543 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,543 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure f2783381a48c13c2f75c07e5029903cb, server=e611192d6313,38729,1731708868111 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,543 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2e79a8ce6f9715045d056d3dfdef5cec, server=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,543 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure c56d152c07a95a189ac65341ff4db1fa, server=e611192d6313,38729,1731708868111 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,543 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2a159d388ea2138ee8e2bed061a785ce, server=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,543 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06b9df8f55d791077e9f7ec55ef6a971, server=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,543 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=81, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 819421a6406c0ebb5635c399b2c0303d, server=e611192d6313,38729,1731708868111 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,543 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=82, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 897a7bb57a002d365489ce583eb7f84f, server=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,544 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 14 msec 2024-11-15T22:14:41,546 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=65 2024-11-15T22:14:41,546 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=e611192d6313,38729,1731708868111, table=hbase:meta, region=1588230740 2024-11-15T22:14:41,546 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e611192d6313,38729,1731708868111, state=OPEN 2024-11-15T22:14:41,567 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open test,kkk,1731708873681.2a159d388ea2138ee8e2bed061a785ce. 2024-11-15T22:14:41,567 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => 2a159d388ea2138ee8e2bed061a785ce, NAME => 'test,kkk,1731708873681.2a159d388ea2138ee8e2bed061a785ce.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-11-15T22:14:41,567 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 2a159d388ea2138ee8e2bed061a785ce 2024-11-15T22:14:41,567 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated test,kkk,1731708873681.2a159d388ea2138ee8e2bed061a785ce.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:41,567 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for 2a159d388ea2138ee8e2bed061a785ce 2024-11-15T22:14:41,567 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for 2a159d388ea2138ee8e2bed061a785ce 2024-11-15T22:14:41,569 INFO [StoreOpener-2a159d388ea2138ee8e2bed061a785ce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 2a159d388ea2138ee8e2bed061a785ce 2024-11-15T22:14:41,569 INFO [StoreOpener-2a159d388ea2138ee8e2bed061a785ce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2a159d388ea2138ee8e2bed061a785ce columnFamilyName family 2024-11-15T22:14:41,569 DEBUG [StoreOpener-2a159d388ea2138ee8e2bed061a785ce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:41,570 INFO [StoreOpener-2a159d388ea2138ee8e2bed061a785ce-1 {}] regionserver.HStore(327): Store=2a159d388ea2138ee8e2bed061a785ce/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:14:41,570 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for 2a159d388ea2138ee8e2bed061a785ce 2024-11-15T22:14:41,571 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/2a159d388ea2138ee8e2bed061a785ce 2024-11-15T22:14:41,572 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/2a159d388ea2138ee8e2bed061a785ce 2024-11-15T22:14:41,572 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for 2a159d388ea2138ee8e2bed061a785ce 2024-11-15T22:14:41,572 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for 2a159d388ea2138ee8e2bed061a785ce 2024-11-15T22:14:41,574 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for 2a159d388ea2138ee8e2bed061a785ce 2024-11-15T22:14:41,575 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened 2a159d388ea2138ee8e2bed061a785ce; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74391329, jitterRate=0.10851718485355377}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T22:14:41,575 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2a159d388ea2138ee8e2bed061a785ce 2024-11-15T22:14:41,575 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for 2a159d388ea2138ee8e2bed061a785ce: Running coprocessor pre-open hook at 1731708881567Writing region info on filesystem at 1731708881567Initializing all the Stores at 1731708881568 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708881568Cleaning up temporary data from old regions at 1731708881572 (+4 ms)Running coprocessor post-open hooks at 1731708881575 (+3 ms)Region opened successfully at 1731708881575 2024-11-15T22:14:41,576 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1731708873681.2a159d388ea2138ee8e2bed061a785ce., pid=79, masterSystemTime=1731708881563 2024-11-15T22:14:41,577 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39087 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:14:41,595 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "e611192d6313" port: 41231 start_code: 1731708869041 } transition { transition_code: OPENED region_info { region_id: 1731708873681 table_name { namespace: "default" qualifier: "test" } start_key: "kkk" end_key: "lll" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 79 initiating_master_active_time: 1731708870437 }; retry (#0) after 1004ms delay (Master is coming online...). org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-15T22:14:41,602 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39087 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:14:41,603 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39087 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:14:41,625 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39087-0x101409049d30000, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:41,625 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x101409049d30001, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:41,625 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41231-0x101409049d30002, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:41,626 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:41,626 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:41,626 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:41,627 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN to state=OFFLINE, location=null, table=test, region=819421a6406c0ebb5635c399b2c0303d to restore RIT 2024-11-15T22:14:41,627 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN to state=OFFLINE, location=null, table=test, region=2a159d388ea2138ee8e2bed061a785ce to restore RIT 2024-11-15T22:14:41,627 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN to state=OFFLINE, location=null, table=test, region=13c3457b9645c9a8181ed9b4551962ad to restore RIT 2024-11-15T22:14:41,627 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f2783381a48c13c2f75c07e5029903cb, ASSIGN to state=OFFLINE, location=null, table=test, region=f2783381a48c13c2f75c07e5029903cb to restore RIT 2024-11-15T22:14:41,627 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06b9df8f55d791077e9f7ec55ef6a971, ASSIGN to state=OFFLINE, location=null, table=test, region=06b9df8f55d791077e9f7ec55ef6a971 to restore RIT 2024-11-15T22:14:41,627 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=897a7bb57a002d365489ce583eb7f84f, ASSIGN to state=OFFLINE, location=null, table=test, region=897a7bb57a002d365489ce583eb7f84f to restore RIT 2024-11-15T22:14:41,627 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c56d152c07a95a189ac65341ff4db1fa, ASSIGN to state=OFFLINE, location=null, table=test, region=c56d152c07a95a189ac65341ff4db1fa to restore RIT 2024-11-15T22:14:41,627 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e79a8ce6f9715045d056d3dfdef5cec, ASSIGN to state=OFFLINE, location=null, table=test, region=2e79a8ce6f9715045d056d3dfdef5cec to restore RIT 2024-11-15T22:14:41,628 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731708911628 2024-11-15T22:14:41,628 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-15T22:14:41,628 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:14:41,629 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-15T22:14:41,630 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-15T22:14:41,631 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=81, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 819421a6406c0ebb5635c399b2c0303d, server=e611192d6313,38729,1731708868111 2024-11-15T22:14:41,631 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T22:14:41,631 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=81, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 819421a6406c0ebb5635c399b2c0303d, server=e611192d6313,38729,1731708868111 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:14:41,656 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f2783381a48c13c2f75c07e5029903cb, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,656 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=897a7bb57a002d365489ce583eb7f84f, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,657 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2424ff4b6d907aafacef6f661d88ef95, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,657 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51bf57bcf1afa9f625176363284d17f7, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,657 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93deaad3e3229244196e8daa9478371a, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=5 2024-11-15T22:14:41,657 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ecaeddccb9b2987f43bb050f0a105976, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,657 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1959cb95bb21ecdc677f7f5290173984, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,657 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13c3457b9645c9a8181ed9b4551962ad, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,658 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1732b30f3d981289760d152f788aff6f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,658 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36b8b3bed3adb94f5901f1f3809b64fa, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,658 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2a159d388ea2138ee8e2bed061a785ce, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,658 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de9977b187fcaabda585b015a568e905, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,658 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f64d8a749084ba25ea1b77db9a0d9f17, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,658 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c56d152c07a95a189ac65341ff4db1fa, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,659 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a602f0e8bd42c685584b8b985174ddea, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,659 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83048e0be415b190a19f64bc2e49d3fe, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,659 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06b9df8f55d791077e9f7ec55ef6a971, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,659 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7502fc847b4b97e1ace9a22ec1c068e0, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,659 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=819421a6406c0ebb5635c399b2c0303d, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,659 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4381d14830a288bee3aeb42195dd8768, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,660 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e87a82b283b7251342a664ba3876c98e, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,660 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9e6155d6c82d9d9a5c9449be6d7ce5f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,660 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e79a8ce6f9715045d056d3dfdef5cec, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,660 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7fb991e3993a690fe7a0a1b064d5f5d, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,660 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1cecce90107a4c37b7d91651975993f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,660 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6efdb15046141ab0cd574540ef2ac2c2, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,660 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-15T22:14:41,661 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731708941661 2024-11-15T22:14:41,661 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731709001661 2024-11-15T22:14:41,661 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 32 msec 2024-11-15T22:14:41,661 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T22:14:41,661 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T22:14:41,661 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T22:14:41,661 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T22:14:41,661 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T22:14:41,661 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T22:14:41,661 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T22:14:41,661 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T22:14:41,662 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-15T22:14:41,666 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-15T22:14:41,666 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T22:14:41,666 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T22:14:41,666 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,666 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=81, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T22:14:41,666 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,666 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T22:14:41,667 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,667 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T22:14:41,667 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,667 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T22:14:41,667 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,667 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T22:14:41,667 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,667 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,667 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T22:14:41,667 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,667 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T22:14:41,667 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T22:14:41,667 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T22:14:41,667 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,667 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T22:14:41,667 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T22:14:41,668 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-15T22:14:41,669 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=93deaad3e3229244196e8daa9478371a, REOPEN/MOVE 2024-11-15T22:14:41,671 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true 2024-11-15T22:14:41,671 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:41,672 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:41,672 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T22:14:41,672 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e611192d6313,38729,1731708868111 2024-11-15T22:14:41,672 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:41,672 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:41,672 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN 2024-11-15T22:14:41,672 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN 2024-11-15T22:14:41,672 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN 2024-11-15T22:14:41,672 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f2783381a48c13c2f75c07e5029903cb, ASSIGN 2024-11-15T22:14:41,672 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06b9df8f55d791077e9f7ec55ef6a971, ASSIGN 2024-11-15T22:14:41,672 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=897a7bb57a002d365489ce583eb7f84f, ASSIGN 2024-11-15T22:14:41,672 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c56d152c07a95a189ac65341ff4db1fa, ASSIGN 2024-11-15T22:14:41,672 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e79a8ce6f9715045d056d3dfdef5cec, ASSIGN 2024-11-15T22:14:41,672 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111 2024-11-15T22:14:41,672 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure f2783381a48c13c2f75c07e5029903cb, server=e611192d6313,38729,1731708868111 2024-11-15T22:14:41,672 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2e79a8ce6f9715045d056d3dfdef5cec, server=e611192d6313,41231,1731708869041 2024-11-15T22:14:41,672 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure c56d152c07a95a189ac65341ff4db1fa, server=e611192d6313,38729,1731708868111 2024-11-15T22:14:41,672 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2a159d388ea2138ee8e2bed061a785ce, server=e611192d6313,41231,1731708869041 2024-11-15T22:14:41,672 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06b9df8f55d791077e9f7ec55ef6a971, server=e611192d6313,41231,1731708869041 2024-11-15T22:14:41,672 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=81, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 819421a6406c0ebb5635c399b2c0303d, server=e611192d6313,38729,1731708868111 2024-11-15T22:14:41,673 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,673 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,673 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,673 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,673 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e611192d6313,38729,1731708868111 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,673 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,673 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,673 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,673 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN 2024-11-15T22:14:41,673 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,673 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN 2024-11-15T22:14:41,673 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,673 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN 2024-11-15T22:14:41,673 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f2783381a48c13c2f75c07e5029903cb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,673 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f2783381a48c13c2f75c07e5029903cb, ASSIGN 2024-11-15T22:14:41,673 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06b9df8f55d791077e9f7ec55ef6a971, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,673 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06b9df8f55d791077e9f7ec55ef6a971, ASSIGN 2024-11-15T22:14:41,673 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=897a7bb57a002d365489ce583eb7f84f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,673 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=897a7bb57a002d365489ce583eb7f84f, ASSIGN 2024-11-15T22:14:41,673 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c56d152c07a95a189ac65341ff4db1fa, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,673 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c56d152c07a95a189ac65341ff4db1fa, ASSIGN 2024-11-15T22:14:41,673 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e79a8ce6f9715045d056d3dfdef5cec, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,673 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e79a8ce6f9715045d056d3dfdef5cec, ASSIGN 2024-11-15T22:14:41,673 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,673 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure f2783381a48c13c2f75c07e5029903cb, server=e611192d6313,38729,1731708868111 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,674 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2e79a8ce6f9715045d056d3dfdef5cec, server=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,674 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure c56d152c07a95a189ac65341ff4db1fa, server=e611192d6313,38729,1731708868111 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,674 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2a159d388ea2138ee8e2bed061a785ce, server=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,674 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06b9df8f55d791077e9f7ec55ef6a971, server=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,674 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=81, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 819421a6406c0ebb5635c399b2c0303d, server=e611192d6313,38729,1731708868111 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,674 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 11 msec 2024-11-15T22:14:41,675 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=65 2024-11-15T22:14:41,675 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=e611192d6313,38729,1731708868111, table=hbase:meta, region=1588230740 2024-11-15T22:14:41,675 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e611192d6313,38729,1731708868111, state=OPEN 2024-11-15T22:14:41,684 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41231-0x101409049d30002, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:41,684 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39087-0x101409049d30000, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:41,684 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x101409049d30001, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:41,684 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:41,684 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:41,684 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:41,685 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN to state=OFFLINE, location=null, table=test, region=819421a6406c0ebb5635c399b2c0303d to restore RIT 2024-11-15T22:14:41,685 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN to state=OFFLINE, location=null, table=test, region=2a159d388ea2138ee8e2bed061a785ce to restore RIT 2024-11-15T22:14:41,685 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN to state=OFFLINE, location=null, table=test, region=13c3457b9645c9a8181ed9b4551962ad to restore RIT 2024-11-15T22:14:41,685 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f2783381a48c13c2f75c07e5029903cb, ASSIGN to state=OFFLINE, location=null, table=test, region=f2783381a48c13c2f75c07e5029903cb to restore RIT 2024-11-15T22:14:41,685 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06b9df8f55d791077e9f7ec55ef6a971, ASSIGN to state=OFFLINE, location=null, table=test, region=06b9df8f55d791077e9f7ec55ef6a971 to restore RIT 2024-11-15T22:14:41,685 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=897a7bb57a002d365489ce583eb7f84f, ASSIGN to state=OFFLINE, location=null, table=test, region=897a7bb57a002d365489ce583eb7f84f to restore RIT 2024-11-15T22:14:41,685 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c56d152c07a95a189ac65341ff4db1fa, ASSIGN to state=OFFLINE, location=null, table=test, region=c56d152c07a95a189ac65341ff4db1fa to restore RIT 2024-11-15T22:14:41,685 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e79a8ce6f9715045d056d3dfdef5cec, ASSIGN to state=OFFLINE, location=null, table=test, region=2e79a8ce6f9715045d056d3dfdef5cec to restore RIT 2024-11-15T22:14:41,685 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731708911685 2024-11-15T22:14:41,685 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-15T22:14:41,686 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:14:41,687 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-15T22:14:41,688 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-15T22:14:41,688 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=80, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06b9df8f55d791077e9f7ec55ef6a971, server=e611192d6313,41231,1731708869041 2024-11-15T22:14:41,688 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T22:14:41,688 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=80, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06b9df8f55d791077e9f7ec55ef6a971, server=e611192d6313,41231,1731708869041 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:14:41,695 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f2783381a48c13c2f75c07e5029903cb, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,696 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=897a7bb57a002d365489ce583eb7f84f, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,696 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2424ff4b6d907aafacef6f661d88ef95, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,696 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51bf57bcf1afa9f625176363284d17f7, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,696 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93deaad3e3229244196e8daa9478371a, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=5 2024-11-15T22:14:41,696 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ecaeddccb9b2987f43bb050f0a105976, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,697 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1959cb95bb21ecdc677f7f5290173984, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,697 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13c3457b9645c9a8181ed9b4551962ad, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,697 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1732b30f3d981289760d152f788aff6f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,697 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36b8b3bed3adb94f5901f1f3809b64fa, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,697 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2a159d388ea2138ee8e2bed061a785ce, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,697 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de9977b187fcaabda585b015a568e905, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,697 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f64d8a749084ba25ea1b77db9a0d9f17, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,698 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c56d152c07a95a189ac65341ff4db1fa, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,698 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a602f0e8bd42c685584b8b985174ddea, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,698 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83048e0be415b190a19f64bc2e49d3fe, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,698 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06b9df8f55d791077e9f7ec55ef6a971, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,698 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7502fc847b4b97e1ace9a22ec1c068e0, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,698 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=819421a6406c0ebb5635c399b2c0303d, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,698 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4381d14830a288bee3aeb42195dd8768, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,699 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e87a82b283b7251342a664ba3876c98e, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,699 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9e6155d6c82d9d9a5c9449be6d7ce5f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,699 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e79a8ce6f9715045d056d3dfdef5cec, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,699 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7fb991e3993a690fe7a0a1b064d5f5d, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,699 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1cecce90107a4c37b7d91651975993f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,699 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6efdb15046141ab0cd574540ef2ac2c2, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,699 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-15T22:14:41,699 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731708941699 2024-11-15T22:14:41,699 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731709001699 2024-11-15T22:14:41,699 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 13 msec 2024-11-15T22:14:41,700 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T22:14:41,700 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T22:14:41,700 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T22:14:41,700 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T22:14:41,700 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T22:14:41,700 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T22:14:41,700 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T22:14:41,700 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T22:14:41,702 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-15T22:14:41,707 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-15T22:14:41,707 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T22:14:41,707 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T22:14:41,707 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,707 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,707 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T22:14:41,707 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,708 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T22:14:41,708 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,708 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T22:14:41,708 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,708 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T22:14:41,708 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,708 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,708 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T22:14:41,708 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,708 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T22:14:41,708 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T22:14:41,708 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T22:14:41,708 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,708 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T22:14:41,709 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T22:14:41,710 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-15T22:14:41,710 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=93deaad3e3229244196e8daa9478371a, REOPEN/MOVE 2024-11-15T22:14:41,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true 2024-11-15T22:14:41,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:41,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:41,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T22:14:41,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e611192d6313,38729,1731708868111 2024-11-15T22:14:41,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:41,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:41,713 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN 2024-11-15T22:14:41,714 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN 2024-11-15T22:14:41,714 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN 2024-11-15T22:14:41,714 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f2783381a48c13c2f75c07e5029903cb, ASSIGN 2024-11-15T22:14:41,714 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06b9df8f55d791077e9f7ec55ef6a971, ASSIGN 2024-11-15T22:14:41,714 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=897a7bb57a002d365489ce583eb7f84f, ASSIGN 2024-11-15T22:14:41,714 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c56d152c07a95a189ac65341ff4db1fa, ASSIGN 2024-11-15T22:14:41,714 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e79a8ce6f9715045d056d3dfdef5cec, ASSIGN 2024-11-15T22:14:41,714 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111 2024-11-15T22:14:41,714 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure f2783381a48c13c2f75c07e5029903cb, server=e611192d6313,38729,1731708868111 2024-11-15T22:14:41,714 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2e79a8ce6f9715045d056d3dfdef5cec, server=e611192d6313,41231,1731708869041 2024-11-15T22:14:41,714 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure c56d152c07a95a189ac65341ff4db1fa, server=e611192d6313,38729,1731708868111 2024-11-15T22:14:41,714 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2a159d388ea2138ee8e2bed061a785ce, server=e611192d6313,41231,1731708869041 2024-11-15T22:14:41,714 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06b9df8f55d791077e9f7ec55ef6a971, server=e611192d6313,41231,1731708869041 2024-11-15T22:14:41,714 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,714 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,714 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,714 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,714 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e611192d6313,38729,1731708868111 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,715 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,715 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,715 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,715 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN 2024-11-15T22:14:41,715 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,715 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN 2024-11-15T22:14:41,715 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,715 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN 2024-11-15T22:14:41,715 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f2783381a48c13c2f75c07e5029903cb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,715 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f2783381a48c13c2f75c07e5029903cb, ASSIGN 2024-11-15T22:14:41,715 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06b9df8f55d791077e9f7ec55ef6a971, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,715 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06b9df8f55d791077e9f7ec55ef6a971, ASSIGN 2024-11-15T22:14:41,715 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=897a7bb57a002d365489ce583eb7f84f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,715 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=897a7bb57a002d365489ce583eb7f84f, ASSIGN 2024-11-15T22:14:41,716 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c56d152c07a95a189ac65341ff4db1fa, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,716 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c56d152c07a95a189ac65341ff4db1fa, ASSIGN 2024-11-15T22:14:41,716 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e79a8ce6f9715045d056d3dfdef5cec, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,716 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e79a8ce6f9715045d056d3dfdef5cec, ASSIGN 2024-11-15T22:14:41,716 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,716 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure f2783381a48c13c2f75c07e5029903cb, server=e611192d6313,38729,1731708868111 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,716 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2e79a8ce6f9715045d056d3dfdef5cec, server=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,716 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure c56d152c07a95a189ac65341ff4db1fa, server=e611192d6313,38729,1731708868111 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,716 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2a159d388ea2138ee8e2bed061a785ce, server=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,716 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06b9df8f55d791077e9f7ec55ef6a971, server=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,716 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 14 msec 2024-11-15T22:14:41,718 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=65 2024-11-15T22:14:41,718 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=e611192d6313,38729,1731708868111, table=hbase:meta, region=1588230740 2024-11-15T22:14:41,718 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e611192d6313,38729,1731708868111, state=OPEN 2024-11-15T22:14:41,727 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39087 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:14:41,728 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39087 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:14:41,730 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39087-0x101409049d30000, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:41,730 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x101409049d30001, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:41,730 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41231-0x101409049d30002, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:41,731 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:41,731 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:41,731 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:41,731 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN to state=OFFLINE, location=null, table=test, region=819421a6406c0ebb5635c399b2c0303d to restore RIT 2024-11-15T22:14:41,731 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN to state=OFFLINE, location=null, table=test, region=2a159d388ea2138ee8e2bed061a785ce to restore RIT 2024-11-15T22:14:41,731 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN to state=OFFLINE, location=null, table=test, region=13c3457b9645c9a8181ed9b4551962ad to restore RIT 2024-11-15T22:14:41,731 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f2783381a48c13c2f75c07e5029903cb, ASSIGN to state=OFFLINE, location=null, table=test, region=f2783381a48c13c2f75c07e5029903cb to restore RIT 2024-11-15T22:14:41,732 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06b9df8f55d791077e9f7ec55ef6a971, ASSIGN to state=OFFLINE, location=null, table=test, region=06b9df8f55d791077e9f7ec55ef6a971 to restore RIT 2024-11-15T22:14:41,732 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=897a7bb57a002d365489ce583eb7f84f, ASSIGN to state=OFFLINE, location=null, table=test, region=897a7bb57a002d365489ce583eb7f84f to restore RIT 2024-11-15T22:14:41,732 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c56d152c07a95a189ac65341ff4db1fa, ASSIGN to state=OFFLINE, location=null, table=test, region=c56d152c07a95a189ac65341ff4db1fa to restore RIT 2024-11-15T22:14:41,732 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e79a8ce6f9715045d056d3dfdef5cec, ASSIGN to state=OFFLINE, location=null, table=test, region=2e79a8ce6f9715045d056d3dfdef5cec to restore RIT 2024-11-15T22:14:41,732 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731708911732 2024-11-15T22:14:41,732 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-15T22:14:41,732 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:14:41,734 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-15T22:14:41,735 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-15T22:14:41,735 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2a159d388ea2138ee8e2bed061a785ce, server=e611192d6313,41231,1731708869041 2024-11-15T22:14:41,735 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T22:14:41,735 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2a159d388ea2138ee8e2bed061a785ce, server=e611192d6313,41231,1731708869041 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:14:41,739 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f2783381a48c13c2f75c07e5029903cb, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,739 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=897a7bb57a002d365489ce583eb7f84f, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,739 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2424ff4b6d907aafacef6f661d88ef95, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,739 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51bf57bcf1afa9f625176363284d17f7, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,740 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93deaad3e3229244196e8daa9478371a, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=5 2024-11-15T22:14:41,740 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ecaeddccb9b2987f43bb050f0a105976, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,740 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1959cb95bb21ecdc677f7f5290173984, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,740 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13c3457b9645c9a8181ed9b4551962ad, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,740 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1732b30f3d981289760d152f788aff6f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,740 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36b8b3bed3adb94f5901f1f3809b64fa, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,740 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2a159d388ea2138ee8e2bed061a785ce, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,741 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de9977b187fcaabda585b015a568e905, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,741 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f64d8a749084ba25ea1b77db9a0d9f17, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,741 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c56d152c07a95a189ac65341ff4db1fa, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,741 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a602f0e8bd42c685584b8b985174ddea, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,741 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83048e0be415b190a19f64bc2e49d3fe, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,741 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06b9df8f55d791077e9f7ec55ef6a971, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,741 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7502fc847b4b97e1ace9a22ec1c068e0, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,742 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=819421a6406c0ebb5635c399b2c0303d, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,742 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4381d14830a288bee3aeb42195dd8768, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,742 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e87a82b283b7251342a664ba3876c98e, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,742 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9e6155d6c82d9d9a5c9449be6d7ce5f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,742 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e79a8ce6f9715045d056d3dfdef5cec, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,742 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7fb991e3993a690fe7a0a1b064d5f5d, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,742 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1cecce90107a4c37b7d91651975993f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,743 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6efdb15046141ab0cd574540ef2ac2c2, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,743 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-15T22:14:41,743 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731708941743 2024-11-15T22:14:41,743 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731709001743 2024-11-15T22:14:41,743 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 10 msec 2024-11-15T22:14:41,743 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T22:14:41,743 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T22:14:41,743 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T22:14:41,743 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T22:14:41,743 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T22:14:41,743 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T22:14:41,743 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T22:14:41,744 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T22:14:41,744 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-15T22:14:41,748 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-15T22:14:41,748 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T22:14:41,748 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T22:14:41,748 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,748 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,748 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T22:14:41,748 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,748 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T22:14:41,748 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,748 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T22:14:41,748 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,748 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,748 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,748 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T22:14:41,748 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,748 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T22:14:41,748 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T22:14:41,748 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T22:14:41,748 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,748 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T22:14:41,749 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T22:14:41,749 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-15T22:14:41,749 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=93deaad3e3229244196e8daa9478371a, REOPEN/MOVE 2024-11-15T22:14:41,752 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true 2024-11-15T22:14:41,752 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:41,752 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:41,752 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T22:14:41,752 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e611192d6313,38729,1731708868111 2024-11-15T22:14:41,752 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:41,752 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:41,752 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN 2024-11-15T22:14:41,752 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN 2024-11-15T22:14:41,752 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN 2024-11-15T22:14:41,752 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f2783381a48c13c2f75c07e5029903cb, ASSIGN 2024-11-15T22:14:41,752 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06b9df8f55d791077e9f7ec55ef6a971, ASSIGN 2024-11-15T22:14:41,752 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=897a7bb57a002d365489ce583eb7f84f, ASSIGN 2024-11-15T22:14:41,752 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c56d152c07a95a189ac65341ff4db1fa, ASSIGN 2024-11-15T22:14:41,752 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e79a8ce6f9715045d056d3dfdef5cec, ASSIGN 2024-11-15T22:14:41,752 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111 2024-11-15T22:14:41,752 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure f2783381a48c13c2f75c07e5029903cb, server=e611192d6313,38729,1731708868111 2024-11-15T22:14:41,752 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2e79a8ce6f9715045d056d3dfdef5cec, server=e611192d6313,41231,1731708869041 2024-11-15T22:14:41,752 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure c56d152c07a95a189ac65341ff4db1fa, server=e611192d6313,38729,1731708868111 2024-11-15T22:14:41,752 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2a159d388ea2138ee8e2bed061a785ce, server=e611192d6313,41231,1731708869041 2024-11-15T22:14:41,752 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,752 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,753 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,753 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,753 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e611192d6313,38729,1731708868111 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,753 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,753 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,753 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,753 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN 2024-11-15T22:14:41,753 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,753 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN 2024-11-15T22:14:41,753 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,753 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN 2024-11-15T22:14:41,753 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f2783381a48c13c2f75c07e5029903cb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,753 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f2783381a48c13c2f75c07e5029903cb, ASSIGN 2024-11-15T22:14:41,753 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06b9df8f55d791077e9f7ec55ef6a971, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,753 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06b9df8f55d791077e9f7ec55ef6a971, ASSIGN 2024-11-15T22:14:41,753 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=897a7bb57a002d365489ce583eb7f84f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,753 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=897a7bb57a002d365489ce583eb7f84f, ASSIGN 2024-11-15T22:14:41,753 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c56d152c07a95a189ac65341ff4db1fa, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,753 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c56d152c07a95a189ac65341ff4db1fa, ASSIGN 2024-11-15T22:14:41,753 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e79a8ce6f9715045d056d3dfdef5cec, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,753 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e79a8ce6f9715045d056d3dfdef5cec, ASSIGN 2024-11-15T22:14:41,753 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,753 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure f2783381a48c13c2f75c07e5029903cb, server=e611192d6313,38729,1731708868111 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,753 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2e79a8ce6f9715045d056d3dfdef5cec, server=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,753 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure c56d152c07a95a189ac65341ff4db1fa, server=e611192d6313,38729,1731708868111 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,753 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2a159d388ea2138ee8e2bed061a785ce, server=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,754 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-15T22:14:41,754 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=65 2024-11-15T22:14:41,755 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=e611192d6313,38729,1731708868111, table=hbase:meta, region=1588230740 2024-11-15T22:14:41,755 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e611192d6313,38729,1731708868111, state=OPEN 2024-11-15T22:14:41,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41231-0x101409049d30002, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:41,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39087-0x101409049d30000, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:41,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x101409049d30001, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:41,822 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:41,822 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:41,822 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:41,823 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN to state=OFFLINE, location=null, table=test, region=819421a6406c0ebb5635c399b2c0303d to restore RIT 2024-11-15T22:14:41,823 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN to state=OFFLINE, location=null, table=test, region=2a159d388ea2138ee8e2bed061a785ce to restore RIT 2024-11-15T22:14:41,823 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN to state=OFFLINE, location=null, table=test, region=13c3457b9645c9a8181ed9b4551962ad to restore RIT 2024-11-15T22:14:41,823 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f2783381a48c13c2f75c07e5029903cb, ASSIGN to state=OFFLINE, location=null, table=test, region=f2783381a48c13c2f75c07e5029903cb to restore RIT 2024-11-15T22:14:41,823 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06b9df8f55d791077e9f7ec55ef6a971, ASSIGN to state=OFFLINE, location=null, table=test, region=06b9df8f55d791077e9f7ec55ef6a971 to restore RIT 2024-11-15T22:14:41,823 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=897a7bb57a002d365489ce583eb7f84f, ASSIGN to state=OFFLINE, location=null, table=test, region=897a7bb57a002d365489ce583eb7f84f to restore RIT 2024-11-15T22:14:41,823 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c56d152c07a95a189ac65341ff4db1fa, ASSIGN to state=OFFLINE, location=null, table=test, region=c56d152c07a95a189ac65341ff4db1fa to restore RIT 2024-11-15T22:14:41,823 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e79a8ce6f9715045d056d3dfdef5cec, ASSIGN to state=OFFLINE, location=null, table=test, region=2e79a8ce6f9715045d056d3dfdef5cec to restore RIT 2024-11-15T22:14:41,824 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731708911824 2024-11-15T22:14:41,824 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-15T22:14:41,824 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:14:41,826 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-15T22:14:41,827 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-15T22:14:41,827 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=78, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure c56d152c07a95a189ac65341ff4db1fa, server=e611192d6313,38729,1731708868111 2024-11-15T22:14:41,827 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T22:14:41,827 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=78, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure c56d152c07a95a189ac65341ff4db1fa, server=e611192d6313,38729,1731708868111 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:14:41,830 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f2783381a48c13c2f75c07e5029903cb, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,831 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=897a7bb57a002d365489ce583eb7f84f, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,831 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2424ff4b6d907aafacef6f661d88ef95, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,831 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51bf57bcf1afa9f625176363284d17f7, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,831 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93deaad3e3229244196e8daa9478371a, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=5 2024-11-15T22:14:41,831 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ecaeddccb9b2987f43bb050f0a105976, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,832 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1959cb95bb21ecdc677f7f5290173984, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,832 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13c3457b9645c9a8181ed9b4551962ad, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,832 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1732b30f3d981289760d152f788aff6f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,832 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36b8b3bed3adb94f5901f1f3809b64fa, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,832 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2a159d388ea2138ee8e2bed061a785ce, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,833 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de9977b187fcaabda585b015a568e905, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,833 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f64d8a749084ba25ea1b77db9a0d9f17, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,833 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c56d152c07a95a189ac65341ff4db1fa, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,833 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a602f0e8bd42c685584b8b985174ddea, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,833 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83048e0be415b190a19f64bc2e49d3fe, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,834 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06b9df8f55d791077e9f7ec55ef6a971, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,834 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7502fc847b4b97e1ace9a22ec1c068e0, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,834 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=819421a6406c0ebb5635c399b2c0303d, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,834 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4381d14830a288bee3aeb42195dd8768, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,834 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e87a82b283b7251342a664ba3876c98e, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,835 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9e6155d6c82d9d9a5c9449be6d7ce5f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,835 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e79a8ce6f9715045d056d3dfdef5cec, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,835 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7fb991e3993a690fe7a0a1b064d5f5d, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,835 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1cecce90107a4c37b7d91651975993f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,835 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6efdb15046141ab0cd574540ef2ac2c2, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,835 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-15T22:14:41,836 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731708941835 2024-11-15T22:14:41,836 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731709001836 2024-11-15T22:14:41,836 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 11 msec 2024-11-15T22:14:41,836 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T22:14:41,836 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T22:14:41,836 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T22:14:41,836 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T22:14:41,837 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T22:14:41,837 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T22:14:41,837 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T22:14:41,837 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T22:14:41,838 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-15T22:14:41,842 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-15T22:14:41,842 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T22:14:41,842 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T22:14:41,842 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,842 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,842 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,842 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T22:14:41,842 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,842 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T22:14:41,842 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,842 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,842 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,843 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T22:14:41,843 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,843 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T22:14:41,843 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T22:14:41,843 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T22:14:41,843 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,843 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T22:14:41,843 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T22:14:41,844 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-15T22:14:41,844 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=93deaad3e3229244196e8daa9478371a, REOPEN/MOVE 2024-11-15T22:14:41,846 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true 2024-11-15T22:14:41,847 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:41,847 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:41,847 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T22:14:41,847 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e611192d6313,38729,1731708868111 2024-11-15T22:14:41,847 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:41,847 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:41,847 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN 2024-11-15T22:14:41,847 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN 2024-11-15T22:14:41,847 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN 2024-11-15T22:14:41,847 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f2783381a48c13c2f75c07e5029903cb, ASSIGN 2024-11-15T22:14:41,847 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06b9df8f55d791077e9f7ec55ef6a971, ASSIGN 2024-11-15T22:14:41,847 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=897a7bb57a002d365489ce583eb7f84f, ASSIGN 2024-11-15T22:14:41,847 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c56d152c07a95a189ac65341ff4db1fa, ASSIGN 2024-11-15T22:14:41,847 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e79a8ce6f9715045d056d3dfdef5cec, ASSIGN 2024-11-15T22:14:41,847 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111 2024-11-15T22:14:41,847 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure f2783381a48c13c2f75c07e5029903cb, server=e611192d6313,38729,1731708868111 2024-11-15T22:14:41,847 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2e79a8ce6f9715045d056d3dfdef5cec, server=e611192d6313,41231,1731708869041 2024-11-15T22:14:41,848 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure c56d152c07a95a189ac65341ff4db1fa, server=e611192d6313,38729,1731708868111 2024-11-15T22:14:41,848 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,848 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,848 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,848 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,848 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e611192d6313,38729,1731708868111 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,848 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,848 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,848 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,848 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN 2024-11-15T22:14:41,848 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,848 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN 2024-11-15T22:14:41,848 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,848 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN 2024-11-15T22:14:41,848 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f2783381a48c13c2f75c07e5029903cb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,848 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f2783381a48c13c2f75c07e5029903cb, ASSIGN 2024-11-15T22:14:41,849 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06b9df8f55d791077e9f7ec55ef6a971, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,849 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06b9df8f55d791077e9f7ec55ef6a971, ASSIGN 2024-11-15T22:14:41,849 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=897a7bb57a002d365489ce583eb7f84f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,849 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=897a7bb57a002d365489ce583eb7f84f, ASSIGN 2024-11-15T22:14:41,849 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c56d152c07a95a189ac65341ff4db1fa, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,849 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c56d152c07a95a189ac65341ff4db1fa, ASSIGN 2024-11-15T22:14:41,849 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e79a8ce6f9715045d056d3dfdef5cec, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,849 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e79a8ce6f9715045d056d3dfdef5cec, ASSIGN 2024-11-15T22:14:41,849 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,849 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure f2783381a48c13c2f75c07e5029903cb, server=e611192d6313,38729,1731708868111 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,849 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2e79a8ce6f9715045d056d3dfdef5cec, server=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,849 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure c56d152c07a95a189ac65341ff4db1fa, server=e611192d6313,38729,1731708868111 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,849 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 11 msec 2024-11-15T22:14:41,850 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=65 2024-11-15T22:14:41,850 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=e611192d6313,38729,1731708868111, table=hbase:meta, region=1588230740 2024-11-15T22:14:41,851 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e611192d6313,38729,1731708868111, state=OPEN 2024-11-15T22:14:41,878 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x101409049d30001, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:41,878 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39087-0x101409049d30000, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:41,878 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41231-0x101409049d30002, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:41,878 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:41,878 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:41,878 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:41,879 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN to state=OFFLINE, location=null, table=test, region=819421a6406c0ebb5635c399b2c0303d to restore RIT 2024-11-15T22:14:41,879 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN to state=OFFLINE, location=null, table=test, region=2a159d388ea2138ee8e2bed061a785ce to restore RIT 2024-11-15T22:14:41,879 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN to state=OFFLINE, location=null, table=test, region=13c3457b9645c9a8181ed9b4551962ad to restore RIT 2024-11-15T22:14:41,879 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f2783381a48c13c2f75c07e5029903cb, ASSIGN to state=OFFLINE, location=null, table=test, region=f2783381a48c13c2f75c07e5029903cb to restore RIT 2024-11-15T22:14:41,879 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06b9df8f55d791077e9f7ec55ef6a971, ASSIGN to state=OFFLINE, location=null, table=test, region=06b9df8f55d791077e9f7ec55ef6a971 to restore RIT 2024-11-15T22:14:41,879 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=897a7bb57a002d365489ce583eb7f84f, ASSIGN to state=OFFLINE, location=null, table=test, region=897a7bb57a002d365489ce583eb7f84f to restore RIT 2024-11-15T22:14:41,879 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c56d152c07a95a189ac65341ff4db1fa, ASSIGN to state=OFFLINE, location=null, table=test, region=c56d152c07a95a189ac65341ff4db1fa to restore RIT 2024-11-15T22:14:41,879 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e79a8ce6f9715045d056d3dfdef5cec, ASSIGN to state=OFFLINE, location=null, table=test, region=2e79a8ce6f9715045d056d3dfdef5cec to restore RIT 2024-11-15T22:14:41,880 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731708911880 2024-11-15T22:14:41,880 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-15T22:14:41,880 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:14:41,881 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-15T22:14:41,882 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-15T22:14:41,882 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2e79a8ce6f9715045d056d3dfdef5cec, server=e611192d6313,41231,1731708869041 2024-11-15T22:14:41,882 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T22:14:41,883 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2e79a8ce6f9715045d056d3dfdef5cec, server=e611192d6313,41231,1731708869041 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:14:41,886 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f2783381a48c13c2f75c07e5029903cb, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,887 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=897a7bb57a002d365489ce583eb7f84f, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,887 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2424ff4b6d907aafacef6f661d88ef95, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,887 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51bf57bcf1afa9f625176363284d17f7, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,887 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93deaad3e3229244196e8daa9478371a, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=5 2024-11-15T22:14:41,887 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ecaeddccb9b2987f43bb050f0a105976, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,887 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1959cb95bb21ecdc677f7f5290173984, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,888 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13c3457b9645c9a8181ed9b4551962ad, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,888 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1732b30f3d981289760d152f788aff6f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,888 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36b8b3bed3adb94f5901f1f3809b64fa, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,888 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2a159d388ea2138ee8e2bed061a785ce, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,888 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de9977b187fcaabda585b015a568e905, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,888 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f64d8a749084ba25ea1b77db9a0d9f17, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,888 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c56d152c07a95a189ac65341ff4db1fa, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,889 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a602f0e8bd42c685584b8b985174ddea, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,889 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83048e0be415b190a19f64bc2e49d3fe, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,889 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06b9df8f55d791077e9f7ec55ef6a971, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,889 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7502fc847b4b97e1ace9a22ec1c068e0, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,889 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=819421a6406c0ebb5635c399b2c0303d, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,889 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4381d14830a288bee3aeb42195dd8768, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,889 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e87a82b283b7251342a664ba3876c98e, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,889 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9e6155d6c82d9d9a5c9449be6d7ce5f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,890 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e79a8ce6f9715045d056d3dfdef5cec, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,890 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7fb991e3993a690fe7a0a1b064d5f5d, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,890 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1cecce90107a4c37b7d91651975993f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,890 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6efdb15046141ab0cd574540ef2ac2c2, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,890 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-15T22:14:41,890 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731708941890 2024-11-15T22:14:41,890 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731709001890 2024-11-15T22:14:41,890 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 10 msec 2024-11-15T22:14:41,890 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T22:14:41,890 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T22:14:41,890 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T22:14:41,890 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T22:14:41,891 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T22:14:41,891 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T22:14:41,891 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T22:14:41,891 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T22:14:41,892 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-15T22:14:41,894 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-15T22:14:41,894 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T22:14:41,894 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T22:14:41,894 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,894 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,894 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,894 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T22:14:41,894 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,894 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T22:14:41,894 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,894 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,894 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,895 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,895 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T22:14:41,895 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T22:14:41,895 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T22:14:41,895 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,895 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T22:14:41,895 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T22:14:41,895 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-15T22:14:41,896 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=93deaad3e3229244196e8daa9478371a, REOPEN/MOVE 2024-11-15T22:14:41,897 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true 2024-11-15T22:14:41,897 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:41,897 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:41,897 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T22:14:41,897 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e611192d6313,38729,1731708868111 2024-11-15T22:14:41,897 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:41,897 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:41,897 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN 2024-11-15T22:14:41,897 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN 2024-11-15T22:14:41,897 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN 2024-11-15T22:14:41,897 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f2783381a48c13c2f75c07e5029903cb, ASSIGN 2024-11-15T22:14:41,897 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06b9df8f55d791077e9f7ec55ef6a971, ASSIGN 2024-11-15T22:14:41,897 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=897a7bb57a002d365489ce583eb7f84f, ASSIGN 2024-11-15T22:14:41,897 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c56d152c07a95a189ac65341ff4db1fa, ASSIGN 2024-11-15T22:14:41,897 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e79a8ce6f9715045d056d3dfdef5cec, ASSIGN 2024-11-15T22:14:41,897 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111 2024-11-15T22:14:41,898 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure f2783381a48c13c2f75c07e5029903cb, server=e611192d6313,38729,1731708868111 2024-11-15T22:14:41,898 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2e79a8ce6f9715045d056d3dfdef5cec, server=e611192d6313,41231,1731708869041 2024-11-15T22:14:41,898 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,898 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,898 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,898 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,898 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e611192d6313,38729,1731708868111 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,898 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,898 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,898 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,898 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN 2024-11-15T22:14:41,898 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,898 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN 2024-11-15T22:14:41,898 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,898 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN 2024-11-15T22:14:41,898 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f2783381a48c13c2f75c07e5029903cb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,898 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f2783381a48c13c2f75c07e5029903cb, ASSIGN 2024-11-15T22:14:41,898 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06b9df8f55d791077e9f7ec55ef6a971, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,898 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06b9df8f55d791077e9f7ec55ef6a971, ASSIGN 2024-11-15T22:14:41,898 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=897a7bb57a002d365489ce583eb7f84f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,898 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=897a7bb57a002d365489ce583eb7f84f, ASSIGN 2024-11-15T22:14:41,898 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c56d152c07a95a189ac65341ff4db1fa, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,898 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c56d152c07a95a189ac65341ff4db1fa, ASSIGN 2024-11-15T22:14:41,898 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e79a8ce6f9715045d056d3dfdef5cec, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,898 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e79a8ce6f9715045d056d3dfdef5cec, ASSIGN 2024-11-15T22:14:41,899 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,899 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure f2783381a48c13c2f75c07e5029903cb, server=e611192d6313,38729,1731708868111 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,899 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2e79a8ce6f9715045d056d3dfdef5cec, server=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,899 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-15T22:14:41,900 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=65 2024-11-15T22:14:41,900 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=e611192d6313,38729,1731708868111, table=hbase:meta, region=1588230740 2024-11-15T22:14:41,900 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e611192d6313,38729,1731708868111, state=OPEN 2024-11-15T22:14:41,909 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41231-0x101409049d30002, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:41,909 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39087-0x101409049d30000, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:41,909 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x101409049d30001, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:41,910 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:41,910 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:41,910 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:41,910 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN to state=OFFLINE, location=null, table=test, region=819421a6406c0ebb5635c399b2c0303d to restore RIT 2024-11-15T22:14:41,910 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN to state=OFFLINE, location=null, table=test, region=2a159d388ea2138ee8e2bed061a785ce to restore RIT 2024-11-15T22:14:41,910 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN to state=OFFLINE, location=null, table=test, region=13c3457b9645c9a8181ed9b4551962ad to restore RIT 2024-11-15T22:14:41,910 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f2783381a48c13c2f75c07e5029903cb, ASSIGN to state=OFFLINE, location=null, table=test, region=f2783381a48c13c2f75c07e5029903cb to restore RIT 2024-11-15T22:14:41,910 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06b9df8f55d791077e9f7ec55ef6a971, ASSIGN to state=OFFLINE, location=null, table=test, region=06b9df8f55d791077e9f7ec55ef6a971 to restore RIT 2024-11-15T22:14:41,911 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=897a7bb57a002d365489ce583eb7f84f, ASSIGN to state=OFFLINE, location=null, table=test, region=897a7bb57a002d365489ce583eb7f84f to restore RIT 2024-11-15T22:14:41,911 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c56d152c07a95a189ac65341ff4db1fa, ASSIGN to state=OFFLINE, location=null, table=test, region=c56d152c07a95a189ac65341ff4db1fa to restore RIT 2024-11-15T22:14:41,911 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e79a8ce6f9715045d056d3dfdef5cec, ASSIGN to state=OFFLINE, location=null, table=test, region=2e79a8ce6f9715045d056d3dfdef5cec to restore RIT 2024-11-15T22:14:41,911 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731708911911 2024-11-15T22:14:41,911 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-15T22:14:41,911 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:14:41,912 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-15T22:14:41,914 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-15T22:14:41,914 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure f2783381a48c13c2f75c07e5029903cb, server=e611192d6313,38729,1731708868111 2024-11-15T22:14:41,914 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T22:14:41,914 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure f2783381a48c13c2f75c07e5029903cb, server=e611192d6313,38729,1731708868111 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:14:41,917 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f2783381a48c13c2f75c07e5029903cb, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,917 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=897a7bb57a002d365489ce583eb7f84f, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,918 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2424ff4b6d907aafacef6f661d88ef95, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,918 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51bf57bcf1afa9f625176363284d17f7, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,918 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93deaad3e3229244196e8daa9478371a, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=5 2024-11-15T22:14:41,918 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ecaeddccb9b2987f43bb050f0a105976, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,918 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1959cb95bb21ecdc677f7f5290173984, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,919 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13c3457b9645c9a8181ed9b4551962ad, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,919 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1732b30f3d981289760d152f788aff6f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,919 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36b8b3bed3adb94f5901f1f3809b64fa, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,919 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2a159d388ea2138ee8e2bed061a785ce, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,919 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de9977b187fcaabda585b015a568e905, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,919 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f64d8a749084ba25ea1b77db9a0d9f17, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,919 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c56d152c07a95a189ac65341ff4db1fa, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,919 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a602f0e8bd42c685584b8b985174ddea, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,919 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83048e0be415b190a19f64bc2e49d3fe, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,920 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06b9df8f55d791077e9f7ec55ef6a971, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,920 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7502fc847b4b97e1ace9a22ec1c068e0, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,920 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=819421a6406c0ebb5635c399b2c0303d, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,920 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4381d14830a288bee3aeb42195dd8768, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,920 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e87a82b283b7251342a664ba3876c98e, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,920 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9e6155d6c82d9d9a5c9449be6d7ce5f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,921 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e79a8ce6f9715045d056d3dfdef5cec, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,921 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7fb991e3993a690fe7a0a1b064d5f5d, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,921 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1cecce90107a4c37b7d91651975993f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,921 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6efdb15046141ab0cd574540ef2ac2c2, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,921 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-15T22:14:41,921 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731708941921 2024-11-15T22:14:41,921 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731709001921 2024-11-15T22:14:41,921 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 10 msec 2024-11-15T22:14:41,921 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T22:14:41,921 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T22:14:41,921 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T22:14:41,921 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T22:14:41,922 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T22:14:41,922 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T22:14:41,922 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T22:14:41,922 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T22:14:41,923 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-15T22:14:41,925 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-15T22:14:41,925 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T22:14:41,925 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T22:14:41,925 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,925 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,925 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,925 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T22:14:41,925 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,925 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T22:14:41,925 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,925 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,925 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,925 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,925 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T22:14:41,925 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T22:14:41,925 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,926 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T22:14:41,926 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T22:14:41,926 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-15T22:14:41,926 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=93deaad3e3229244196e8daa9478371a, REOPEN/MOVE 2024-11-15T22:14:41,927 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true 2024-11-15T22:14:41,928 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:41,928 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:41,928 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T22:14:41,928 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e611192d6313,38729,1731708868111 2024-11-15T22:14:41,928 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:41,928 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:41,928 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN 2024-11-15T22:14:41,928 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN 2024-11-15T22:14:41,928 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN 2024-11-15T22:14:41,928 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f2783381a48c13c2f75c07e5029903cb, ASSIGN 2024-11-15T22:14:41,928 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06b9df8f55d791077e9f7ec55ef6a971, ASSIGN 2024-11-15T22:14:41,928 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=897a7bb57a002d365489ce583eb7f84f, ASSIGN 2024-11-15T22:14:41,928 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c56d152c07a95a189ac65341ff4db1fa, ASSIGN 2024-11-15T22:14:41,928 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e79a8ce6f9715045d056d3dfdef5cec, ASSIGN 2024-11-15T22:14:41,928 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111 2024-11-15T22:14:41,928 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure f2783381a48c13c2f75c07e5029903cb, server=e611192d6313,38729,1731708868111 2024-11-15T22:14:41,928 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,928 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,928 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,928 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,928 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e611192d6313,38729,1731708868111 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,928 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,928 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,929 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,929 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN 2024-11-15T22:14:41,929 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,929 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN 2024-11-15T22:14:41,929 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,929 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN 2024-11-15T22:14:41,929 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f2783381a48c13c2f75c07e5029903cb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,929 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f2783381a48c13c2f75c07e5029903cb, ASSIGN 2024-11-15T22:14:41,929 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06b9df8f55d791077e9f7ec55ef6a971, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,929 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06b9df8f55d791077e9f7ec55ef6a971, ASSIGN 2024-11-15T22:14:41,929 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=897a7bb57a002d365489ce583eb7f84f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,929 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=897a7bb57a002d365489ce583eb7f84f, ASSIGN 2024-11-15T22:14:41,929 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c56d152c07a95a189ac65341ff4db1fa, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,929 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c56d152c07a95a189ac65341ff4db1fa, ASSIGN 2024-11-15T22:14:41,929 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e79a8ce6f9715045d056d3dfdef5cec, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,929 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e79a8ce6f9715045d056d3dfdef5cec, ASSIGN 2024-11-15T22:14:41,929 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,929 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure f2783381a48c13c2f75c07e5029903cb, server=e611192d6313,38729,1731708868111 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,929 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-15T22:14:41,930 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=65 2024-11-15T22:14:41,930 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=e611192d6313,38729,1731708868111, table=hbase:meta, region=1588230740 2024-11-15T22:14:41,930 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e611192d6313,38729,1731708868111, state=OPEN 2024-11-15T22:14:41,934 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39087 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:14:41,935 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39087 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:14:41,941 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x101409049d30001, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:41,941 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41231-0x101409049d30002, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:41,941 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39087-0x101409049d30000, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:41,941 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:41,941 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:41,941 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:41,942 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN to state=OFFLINE, location=null, table=test, region=819421a6406c0ebb5635c399b2c0303d to restore RIT 2024-11-15T22:14:41,942 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN to state=OFFLINE, location=null, table=test, region=2a159d388ea2138ee8e2bed061a785ce to restore RIT 2024-11-15T22:14:41,942 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN to state=OFFLINE, location=null, table=test, region=13c3457b9645c9a8181ed9b4551962ad to restore RIT 2024-11-15T22:14:41,942 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f2783381a48c13c2f75c07e5029903cb, ASSIGN to state=OFFLINE, location=null, table=test, region=f2783381a48c13c2f75c07e5029903cb to restore RIT 2024-11-15T22:14:41,942 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06b9df8f55d791077e9f7ec55ef6a971, ASSIGN to state=OFFLINE, location=null, table=test, region=06b9df8f55d791077e9f7ec55ef6a971 to restore RIT 2024-11-15T22:14:41,942 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=897a7bb57a002d365489ce583eb7f84f, ASSIGN to state=OFFLINE, location=null, table=test, region=897a7bb57a002d365489ce583eb7f84f to restore RIT 2024-11-15T22:14:41,942 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c56d152c07a95a189ac65341ff4db1fa, ASSIGN to state=OFFLINE, location=null, table=test, region=c56d152c07a95a189ac65341ff4db1fa to restore RIT 2024-11-15T22:14:41,942 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e79a8ce6f9715045d056d3dfdef5cec, ASSIGN to state=OFFLINE, location=null, table=test, region=2e79a8ce6f9715045d056d3dfdef5cec to restore RIT 2024-11-15T22:14:41,943 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731708911943 2024-11-15T22:14:41,943 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-15T22:14:41,943 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:14:41,944 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-15T22:14:41,945 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-15T22:14:41,945 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=75, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111 2024-11-15T22:14:41,945 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T22:14:41,945 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=75, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:14:41,949 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f2783381a48c13c2f75c07e5029903cb, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,949 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=897a7bb57a002d365489ce583eb7f84f, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,949 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2424ff4b6d907aafacef6f661d88ef95, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,949 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51bf57bcf1afa9f625176363284d17f7, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,949 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93deaad3e3229244196e8daa9478371a, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=5 2024-11-15T22:14:41,950 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ecaeddccb9b2987f43bb050f0a105976, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,950 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1959cb95bb21ecdc677f7f5290173984, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,950 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13c3457b9645c9a8181ed9b4551962ad, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,950 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1732b30f3d981289760d152f788aff6f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,950 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36b8b3bed3adb94f5901f1f3809b64fa, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,950 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2a159d388ea2138ee8e2bed061a785ce, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,951 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de9977b187fcaabda585b015a568e905, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,951 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f64d8a749084ba25ea1b77db9a0d9f17, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,951 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c56d152c07a95a189ac65341ff4db1fa, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,951 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a602f0e8bd42c685584b8b985174ddea, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,951 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83048e0be415b190a19f64bc2e49d3fe, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,951 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06b9df8f55d791077e9f7ec55ef6a971, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,952 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7502fc847b4b97e1ace9a22ec1c068e0, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,952 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=819421a6406c0ebb5635c399b2c0303d, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,952 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4381d14830a288bee3aeb42195dd8768, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,952 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e87a82b283b7251342a664ba3876c98e, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,952 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9e6155d6c82d9d9a5c9449be6d7ce5f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,952 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e79a8ce6f9715045d056d3dfdef5cec, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,953 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7fb991e3993a690fe7a0a1b064d5f5d, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,953 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1cecce90107a4c37b7d91651975993f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,953 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6efdb15046141ab0cd574540ef2ac2c2, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,953 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-15T22:14:41,953 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731708941953 2024-11-15T22:14:41,953 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731709001953 2024-11-15T22:14:41,953 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 10 msec 2024-11-15T22:14:41,953 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T22:14:41,953 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T22:14:41,953 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T22:14:41,954 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T22:14:41,954 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T22:14:41,954 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T22:14:41,954 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T22:14:41,954 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T22:14:41,955 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-15T22:14:41,958 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-15T22:14:41,958 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T22:14:41,958 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T22:14:41,958 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,959 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,959 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,959 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T22:14:41,959 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,959 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,959 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,959 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,959 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,959 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T22:14:41,959 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T22:14:41,959 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,959 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T22:14:41,960 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T22:14:41,960 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-15T22:14:41,961 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=93deaad3e3229244196e8daa9478371a, REOPEN/MOVE 2024-11-15T22:14:41,963 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true 2024-11-15T22:14:41,963 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:41,963 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:41,963 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T22:14:41,963 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e611192d6313,38729,1731708868111 2024-11-15T22:14:41,963 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:41,963 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:41,963 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN 2024-11-15T22:14:41,963 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN 2024-11-15T22:14:41,963 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN 2024-11-15T22:14:41,963 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f2783381a48c13c2f75c07e5029903cb, ASSIGN 2024-11-15T22:14:41,963 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06b9df8f55d791077e9f7ec55ef6a971, ASSIGN 2024-11-15T22:14:41,963 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=897a7bb57a002d365489ce583eb7f84f, ASSIGN 2024-11-15T22:14:41,964 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c56d152c07a95a189ac65341ff4db1fa, ASSIGN 2024-11-15T22:14:41,964 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e79a8ce6f9715045d056d3dfdef5cec, ASSIGN 2024-11-15T22:14:41,964 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111 2024-11-15T22:14:41,964 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,964 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,964 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,964 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,964 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e611192d6313,38729,1731708868111 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,964 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,964 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,964 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,964 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN 2024-11-15T22:14:41,964 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,964 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN 2024-11-15T22:14:41,964 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,964 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN 2024-11-15T22:14:41,964 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f2783381a48c13c2f75c07e5029903cb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,964 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f2783381a48c13c2f75c07e5029903cb, ASSIGN 2024-11-15T22:14:41,965 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06b9df8f55d791077e9f7ec55ef6a971, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,965 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06b9df8f55d791077e9f7ec55ef6a971, ASSIGN 2024-11-15T22:14:41,965 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=897a7bb57a002d365489ce583eb7f84f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,965 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=897a7bb57a002d365489ce583eb7f84f, ASSIGN 2024-11-15T22:14:41,965 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c56d152c07a95a189ac65341ff4db1fa, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,965 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c56d152c07a95a189ac65341ff4db1fa, ASSIGN 2024-11-15T22:14:41,965 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e79a8ce6f9715045d056d3dfdef5cec, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:41,965 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e79a8ce6f9715045d056d3dfdef5cec, ASSIGN 2024-11-15T22:14:41,965 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:41,965 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-15T22:14:41,966 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=65 2024-11-15T22:14:41,966 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=e611192d6313,38729,1731708868111, table=hbase:meta, region=1588230740 2024-11-15T22:14:41,966 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e611192d6313,38729,1731708868111, state=OPEN 2024-11-15T22:14:41,978 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39087-0x101409049d30000, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:41,978 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x101409049d30001, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:41,978 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41231-0x101409049d30002, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:41,979 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:41,979 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:41,979 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:41,979 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN to state=OFFLINE, location=null, table=test, region=819421a6406c0ebb5635c399b2c0303d to restore RIT 2024-11-15T22:14:41,979 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN to state=OFFLINE, location=null, table=test, region=2a159d388ea2138ee8e2bed061a785ce to restore RIT 2024-11-15T22:14:41,979 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN to state=OFFLINE, location=null, table=test, region=13c3457b9645c9a8181ed9b4551962ad to restore RIT 2024-11-15T22:14:41,979 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f2783381a48c13c2f75c07e5029903cb, ASSIGN to state=OFFLINE, location=null, table=test, region=f2783381a48c13c2f75c07e5029903cb to restore RIT 2024-11-15T22:14:41,980 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06b9df8f55d791077e9f7ec55ef6a971, ASSIGN to state=OFFLINE, location=null, table=test, region=06b9df8f55d791077e9f7ec55ef6a971 to restore RIT 2024-11-15T22:14:41,980 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=897a7bb57a002d365489ce583eb7f84f, ASSIGN to state=OFFLINE, location=null, table=test, region=897a7bb57a002d365489ce583eb7f84f to restore RIT 2024-11-15T22:14:41,980 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c56d152c07a95a189ac65341ff4db1fa, ASSIGN to state=OFFLINE, location=null, table=test, region=c56d152c07a95a189ac65341ff4db1fa to restore RIT 2024-11-15T22:14:41,980 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e79a8ce6f9715045d056d3dfdef5cec, ASSIGN to state=OFFLINE, location=null, table=test, region=2e79a8ce6f9715045d056d3dfdef5cec to restore RIT 2024-11-15T22:14:41,980 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731708911980 2024-11-15T22:14:41,980 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-15T22:14:41,980 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:14:41,981 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-15T22:14:41,982 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-15T22:14:41,982 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e79a8ce6f9715045d056d3dfdef5cec, ASSIGN 2024-11-15T22:14:41,982 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T22:14:41,982 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e79a8ce6f9715045d056d3dfdef5cec, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:14:41,985 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f2783381a48c13c2f75c07e5029903cb, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,985 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=897a7bb57a002d365489ce583eb7f84f, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,985 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2424ff4b6d907aafacef6f661d88ef95, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,986 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51bf57bcf1afa9f625176363284d17f7, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,986 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93deaad3e3229244196e8daa9478371a, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=5 2024-11-15T22:14:41,986 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ecaeddccb9b2987f43bb050f0a105976, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,986 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1959cb95bb21ecdc677f7f5290173984, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,986 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13c3457b9645c9a8181ed9b4551962ad, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,986 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1732b30f3d981289760d152f788aff6f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,986 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36b8b3bed3adb94f5901f1f3809b64fa, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,986 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2a159d388ea2138ee8e2bed061a785ce, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,987 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de9977b187fcaabda585b015a568e905, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,987 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f64d8a749084ba25ea1b77db9a0d9f17, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,987 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c56d152c07a95a189ac65341ff4db1fa, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,987 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a602f0e8bd42c685584b8b985174ddea, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,987 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83048e0be415b190a19f64bc2e49d3fe, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,987 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06b9df8f55d791077e9f7ec55ef6a971, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,987 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7502fc847b4b97e1ace9a22ec1c068e0, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,988 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=819421a6406c0ebb5635c399b2c0303d, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,988 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4381d14830a288bee3aeb42195dd8768, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,988 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e87a82b283b7251342a664ba3876c98e, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,988 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9e6155d6c82d9d9a5c9449be6d7ce5f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,988 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e79a8ce6f9715045d056d3dfdef5cec, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,988 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7fb991e3993a690fe7a0a1b064d5f5d, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:41,988 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1cecce90107a4c37b7d91651975993f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,989 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6efdb15046141ab0cd574540ef2ac2c2, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:41,989 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-15T22:14:41,989 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731708941989 2024-11-15T22:14:41,989 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731709001989 2024-11-15T22:14:41,989 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-15T22:14:41,989 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T22:14:41,989 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T22:14:41,989 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T22:14:41,989 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T22:14:41,990 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T22:14:41,990 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T22:14:41,990 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T22:14:41,990 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T22:14:41,996 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 6 msec 2024-11-15T22:14:41,998 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-15T22:14:41,998 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T22:14:41,998 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T22:14:41,998 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,999 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,999 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,999 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,999 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,999 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,999 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,999 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,999 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T22:14:41,999 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T22:14:41,999 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:41,999 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T22:14:41,999 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T22:14:42,000 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-15T22:14:42,000 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=93deaad3e3229244196e8daa9478371a, REOPEN/MOVE 2024-11-15T22:14:42,001 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true 2024-11-15T22:14:42,001 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:42,001 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:42,001 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T22:14:42,001 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e611192d6313,38729,1731708868111 2024-11-15T22:14:42,001 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:42,001 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:42,001 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN 2024-11-15T22:14:42,001 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN 2024-11-15T22:14:42,001 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN 2024-11-15T22:14:42,001 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f2783381a48c13c2f75c07e5029903cb, ASSIGN 2024-11-15T22:14:42,001 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06b9df8f55d791077e9f7ec55ef6a971, ASSIGN 2024-11-15T22:14:42,002 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=897a7bb57a002d365489ce583eb7f84f, ASSIGN 2024-11-15T22:14:42,002 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c56d152c07a95a189ac65341ff4db1fa, ASSIGN 2024-11-15T22:14:42,002 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e79a8ce6f9715045d056d3dfdef5cec, ASSIGN 2024-11-15T22:14:42,002 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:42,002 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,002 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,002 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,002 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e611192d6313,38729,1731708868111 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,002 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,002 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,002 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:42,002 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN 2024-11-15T22:14:42,002 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:42,002 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN 2024-11-15T22:14:42,002 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:42,002 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN 2024-11-15T22:14:42,002 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f2783381a48c13c2f75c07e5029903cb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:42,002 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f2783381a48c13c2f75c07e5029903cb, ASSIGN 2024-11-15T22:14:42,002 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06b9df8f55d791077e9f7ec55ef6a971, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:42,002 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06b9df8f55d791077e9f7ec55ef6a971, ASSIGN 2024-11-15T22:14:42,003 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=897a7bb57a002d365489ce583eb7f84f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:42,003 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=897a7bb57a002d365489ce583eb7f84f, ASSIGN 2024-11-15T22:14:42,003 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c56d152c07a95a189ac65341ff4db1fa, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:42,003 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c56d152c07a95a189ac65341ff4db1fa, ASSIGN 2024-11-15T22:14:42,003 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e79a8ce6f9715045d056d3dfdef5cec, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:42,003 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e79a8ce6f9715045d056d3dfdef5cec, ASSIGN 2024-11-15T22:14:42,003 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-15T22:14:42,004 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=65 2024-11-15T22:14:42,004 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=e611192d6313,38729,1731708868111, table=hbase:meta, region=1588230740 2024-11-15T22:14:42,004 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e611192d6313,38729,1731708868111, state=OPEN 2024-11-15T22:14:42,015 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x101409049d30001, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:42,015 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39087-0x101409049d30000, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:42,015 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41231-0x101409049d30002, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:42,015 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:42,015 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:42,015 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:42,016 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN to state=OFFLINE, location=null, table=test, region=819421a6406c0ebb5635c399b2c0303d to restore RIT 2024-11-15T22:14:42,016 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN to state=OFFLINE, location=null, table=test, region=2a159d388ea2138ee8e2bed061a785ce to restore RIT 2024-11-15T22:14:42,016 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN to state=OFFLINE, location=null, table=test, region=13c3457b9645c9a8181ed9b4551962ad to restore RIT 2024-11-15T22:14:42,016 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f2783381a48c13c2f75c07e5029903cb, ASSIGN to state=OFFLINE, location=null, table=test, region=f2783381a48c13c2f75c07e5029903cb to restore RIT 2024-11-15T22:14:42,016 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06b9df8f55d791077e9f7ec55ef6a971, ASSIGN to state=OFFLINE, location=null, table=test, region=06b9df8f55d791077e9f7ec55ef6a971 to restore RIT 2024-11-15T22:14:42,016 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=897a7bb57a002d365489ce583eb7f84f, ASSIGN to state=OFFLINE, location=null, table=test, region=897a7bb57a002d365489ce583eb7f84f to restore RIT 2024-11-15T22:14:42,017 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c56d152c07a95a189ac65341ff4db1fa, ASSIGN to state=OFFLINE, location=null, table=test, region=c56d152c07a95a189ac65341ff4db1fa to restore RIT 2024-11-15T22:14:42,017 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2e79a8ce6f9715045d056d3dfdef5cec, ASSIGN to state=OFFLINE, location=null, table=test, region=2e79a8ce6f9715045d056d3dfdef5cec to restore RIT 2024-11-15T22:14:42,017 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731708912017 2024-11-15T22:14:42,017 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-15T22:14:42,017 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:14:42,019 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-15T22:14:42,020 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-15T22:14:42,020 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c56d152c07a95a189ac65341ff4db1fa, ASSIGN 2024-11-15T22:14:42,020 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T22:14:42,020 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c56d152c07a95a189ac65341ff4db1fa, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:14:42,023 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f2783381a48c13c2f75c07e5029903cb, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,024 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=897a7bb57a002d365489ce583eb7f84f, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,024 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2424ff4b6d907aafacef6f661d88ef95, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,024 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51bf57bcf1afa9f625176363284d17f7, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,024 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93deaad3e3229244196e8daa9478371a, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=5 2024-11-15T22:14:42,024 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ecaeddccb9b2987f43bb050f0a105976, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,024 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1959cb95bb21ecdc677f7f5290173984, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,024 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13c3457b9645c9a8181ed9b4551962ad, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,024 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1732b30f3d981289760d152f788aff6f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,025 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36b8b3bed3adb94f5901f1f3809b64fa, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,025 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2a159d388ea2138ee8e2bed061a785ce, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,025 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de9977b187fcaabda585b015a568e905, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,025 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f64d8a749084ba25ea1b77db9a0d9f17, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,025 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c56d152c07a95a189ac65341ff4db1fa, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,025 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a602f0e8bd42c685584b8b985174ddea, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,025 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83048e0be415b190a19f64bc2e49d3fe, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,025 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06b9df8f55d791077e9f7ec55ef6a971, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,025 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7502fc847b4b97e1ace9a22ec1c068e0, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,026 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=819421a6406c0ebb5635c399b2c0303d, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,026 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4381d14830a288bee3aeb42195dd8768, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,026 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e87a82b283b7251342a664ba3876c98e, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,026 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9e6155d6c82d9d9a5c9449be6d7ce5f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,026 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e79a8ce6f9715045d056d3dfdef5cec, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,026 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7fb991e3993a690fe7a0a1b064d5f5d, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,026 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1cecce90107a4c37b7d91651975993f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,026 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6efdb15046141ab0cd574540ef2ac2c2, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,026 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-15T22:14:42,026 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731708942026 2024-11-15T22:14:42,026 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731709002026 2024-11-15T22:14:42,026 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-15T22:14:42,027 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T22:14:42,027 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T22:14:42,027 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T22:14:42,027 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T22:14:42,027 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T22:14:42,027 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T22:14:42,027 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T22:14:42,027 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T22:14:42,028 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-15T22:14:42,030 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-15T22:14:42,030 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T22:14:42,030 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T22:14:42,030 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:42,030 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:42,030 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:42,030 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:42,030 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:42,030 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:42,030 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:42,030 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T22:14:42,030 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T22:14:42,030 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:42,030 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T22:14:42,030 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T22:14:42,031 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-15T22:14:42,031 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=93deaad3e3229244196e8daa9478371a, REOPEN/MOVE 2024-11-15T22:14:42,032 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true 2024-11-15T22:14:42,032 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:42,032 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:42,032 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T22:14:42,032 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e611192d6313,38729,1731708868111 2024-11-15T22:14:42,032 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:42,032 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:42,032 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN 2024-11-15T22:14:42,032 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN 2024-11-15T22:14:42,032 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN 2024-11-15T22:14:42,032 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f2783381a48c13c2f75c07e5029903cb, ASSIGN 2024-11-15T22:14:42,032 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06b9df8f55d791077e9f7ec55ef6a971, ASSIGN 2024-11-15T22:14:42,032 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=897a7bb57a002d365489ce583eb7f84f, ASSIGN 2024-11-15T22:14:42,032 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c56d152c07a95a189ac65341ff4db1fa, ASSIGN 2024-11-15T22:14:42,032 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:42,032 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,032 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,032 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,032 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e611192d6313,38729,1731708868111 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,032 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,032 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,033 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:42,033 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN 2024-11-15T22:14:42,033 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:42,033 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN 2024-11-15T22:14:42,033 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:42,033 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN 2024-11-15T22:14:42,033 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f2783381a48c13c2f75c07e5029903cb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:42,033 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f2783381a48c13c2f75c07e5029903cb, ASSIGN 2024-11-15T22:14:42,033 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06b9df8f55d791077e9f7ec55ef6a971, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:42,033 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06b9df8f55d791077e9f7ec55ef6a971, ASSIGN 2024-11-15T22:14:42,033 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=897a7bb57a002d365489ce583eb7f84f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:42,033 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=897a7bb57a002d365489ce583eb7f84f, ASSIGN 2024-11-15T22:14:42,033 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c56d152c07a95a189ac65341ff4db1fa, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:42,033 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c56d152c07a95a189ac65341ff4db1fa, ASSIGN 2024-11-15T22:14:42,033 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-15T22:14:42,034 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=65 2024-11-15T22:14:42,034 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=e611192d6313,38729,1731708868111, table=hbase:meta, region=1588230740 2024-11-15T22:14:42,034 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e611192d6313,38729,1731708868111, state=OPEN 2024-11-15T22:14:42,043 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39087 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:14:42,043 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39087 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:14:42,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39087-0x101409049d30000, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:42,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41231-0x101409049d30002, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:42,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x101409049d30001, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:42,046 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:42,046 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:42,046 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:42,047 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN to state=OFFLINE, location=null, table=test, region=819421a6406c0ebb5635c399b2c0303d to restore RIT 2024-11-15T22:14:42,047 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN to state=OFFLINE, location=null, table=test, region=2a159d388ea2138ee8e2bed061a785ce to restore RIT 2024-11-15T22:14:42,047 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN to state=OFFLINE, location=null, table=test, region=13c3457b9645c9a8181ed9b4551962ad to restore RIT 2024-11-15T22:14:42,047 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f2783381a48c13c2f75c07e5029903cb, ASSIGN to state=OFFLINE, location=null, table=test, region=f2783381a48c13c2f75c07e5029903cb to restore RIT 2024-11-15T22:14:42,047 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06b9df8f55d791077e9f7ec55ef6a971, ASSIGN to state=OFFLINE, location=null, table=test, region=06b9df8f55d791077e9f7ec55ef6a971 to restore RIT 2024-11-15T22:14:42,047 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=897a7bb57a002d365489ce583eb7f84f, ASSIGN to state=OFFLINE, location=null, table=test, region=897a7bb57a002d365489ce583eb7f84f to restore RIT 2024-11-15T22:14:42,047 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c56d152c07a95a189ac65341ff4db1fa, ASSIGN to state=OFFLINE, location=null, table=test, region=c56d152c07a95a189ac65341ff4db1fa to restore RIT 2024-11-15T22:14:42,048 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731708912048 2024-11-15T22:14:42,048 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-15T22:14:42,048 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:14:42,049 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-15T22:14:42,050 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-15T22:14:42,050 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=897a7bb57a002d365489ce583eb7f84f, ASSIGN 2024-11-15T22:14:42,050 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T22:14:42,050 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=897a7bb57a002d365489ce583eb7f84f, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:14:42,053 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f2783381a48c13c2f75c07e5029903cb, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,053 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=897a7bb57a002d365489ce583eb7f84f, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,053 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2424ff4b6d907aafacef6f661d88ef95, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,053 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51bf57bcf1afa9f625176363284d17f7, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,053 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93deaad3e3229244196e8daa9478371a, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=5 2024-11-15T22:14:42,054 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ecaeddccb9b2987f43bb050f0a105976, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,054 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1959cb95bb21ecdc677f7f5290173984, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,054 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13c3457b9645c9a8181ed9b4551962ad, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,054 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1732b30f3d981289760d152f788aff6f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,054 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36b8b3bed3adb94f5901f1f3809b64fa, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,054 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2a159d388ea2138ee8e2bed061a785ce, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,054 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de9977b187fcaabda585b015a568e905, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,054 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f64d8a749084ba25ea1b77db9a0d9f17, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,054 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c56d152c07a95a189ac65341ff4db1fa, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,054 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a602f0e8bd42c685584b8b985174ddea, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,055 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83048e0be415b190a19f64bc2e49d3fe, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,055 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06b9df8f55d791077e9f7ec55ef6a971, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,055 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7502fc847b4b97e1ace9a22ec1c068e0, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,055 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=819421a6406c0ebb5635c399b2c0303d, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,055 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4381d14830a288bee3aeb42195dd8768, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,055 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e87a82b283b7251342a664ba3876c98e, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,055 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9e6155d6c82d9d9a5c9449be6d7ce5f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,055 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e79a8ce6f9715045d056d3dfdef5cec, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,055 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7fb991e3993a690fe7a0a1b064d5f5d, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,055 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1cecce90107a4c37b7d91651975993f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,056 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6efdb15046141ab0cd574540ef2ac2c2, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,056 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-15T22:14:42,056 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731708942056 2024-11-15T22:14:42,056 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731709002056 2024-11-15T22:14:42,056 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 7 msec 2024-11-15T22:14:42,056 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T22:14:42,056 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T22:14:42,056 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T22:14:42,056 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T22:14:42,056 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T22:14:42,056 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T22:14:42,056 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T22:14:42,057 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T22:14:42,057 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-15T22:14:42,059 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-15T22:14:42,059 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T22:14:42,059 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T22:14:42,059 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:42,059 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:42,059 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:42,059 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:42,059 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:42,059 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:42,059 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T22:14:42,059 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T22:14:42,059 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:42,059 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T22:14:42,059 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T22:14:42,060 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-15T22:14:42,060 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=93deaad3e3229244196e8daa9478371a, REOPEN/MOVE 2024-11-15T22:14:42,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true 2024-11-15T22:14:42,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:42,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:42,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T22:14:42,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e611192d6313,38729,1731708868111 2024-11-15T22:14:42,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:42,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:42,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN 2024-11-15T22:14:42,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN 2024-11-15T22:14:42,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN 2024-11-15T22:14:42,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f2783381a48c13c2f75c07e5029903cb, ASSIGN 2024-11-15T22:14:42,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06b9df8f55d791077e9f7ec55ef6a971, ASSIGN 2024-11-15T22:14:42,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=897a7bb57a002d365489ce583eb7f84f, ASSIGN 2024-11-15T22:14:42,061 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:42,061 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,061 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,061 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,061 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e611192d6313,38729,1731708868111 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,061 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,062 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,062 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:42,062 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN 2024-11-15T22:14:42,062 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:42,062 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN 2024-11-15T22:14:42,062 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:42,062 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN 2024-11-15T22:14:42,062 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f2783381a48c13c2f75c07e5029903cb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:42,062 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f2783381a48c13c2f75c07e5029903cb, ASSIGN 2024-11-15T22:14:42,062 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06b9df8f55d791077e9f7ec55ef6a971, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:42,062 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06b9df8f55d791077e9f7ec55ef6a971, ASSIGN 2024-11-15T22:14:42,062 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=897a7bb57a002d365489ce583eb7f84f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:42,062 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=897a7bb57a002d365489ce583eb7f84f, ASSIGN 2024-11-15T22:14:42,062 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-15T22:14:42,063 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=65 2024-11-15T22:14:42,063 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=e611192d6313,38729,1731708868111, table=hbase:meta, region=1588230740 2024-11-15T22:14:42,063 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e611192d6313,38729,1731708868111, state=OPEN 2024-11-15T22:14:42,073 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x101409049d30001, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:42,073 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41231-0x101409049d30002, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:42,073 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39087-0x101409049d30000, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:42,073 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:42,073 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:42,073 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:42,074 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN to state=OFFLINE, location=null, table=test, region=819421a6406c0ebb5635c399b2c0303d to restore RIT 2024-11-15T22:14:42,074 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN to state=OFFLINE, location=null, table=test, region=2a159d388ea2138ee8e2bed061a785ce to restore RIT 2024-11-15T22:14:42,074 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN to state=OFFLINE, location=null, table=test, region=13c3457b9645c9a8181ed9b4551962ad to restore RIT 2024-11-15T22:14:42,074 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f2783381a48c13c2f75c07e5029903cb, ASSIGN to state=OFFLINE, location=null, table=test, region=f2783381a48c13c2f75c07e5029903cb to restore RIT 2024-11-15T22:14:42,074 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06b9df8f55d791077e9f7ec55ef6a971, ASSIGN to state=OFFLINE, location=null, table=test, region=06b9df8f55d791077e9f7ec55ef6a971 to restore RIT 2024-11-15T22:14:42,074 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=897a7bb57a002d365489ce583eb7f84f, ASSIGN to state=OFFLINE, location=null, table=test, region=897a7bb57a002d365489ce583eb7f84f to restore RIT 2024-11-15T22:14:42,075 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731708912075 2024-11-15T22:14:42,075 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-15T22:14:42,075 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:14:42,076 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-15T22:14:42,076 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-15T22:14:42,077 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06b9df8f55d791077e9f7ec55ef6a971, ASSIGN 2024-11-15T22:14:42,077 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T22:14:42,077 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06b9df8f55d791077e9f7ec55ef6a971, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:14:42,079 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f2783381a48c13c2f75c07e5029903cb, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,079 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=897a7bb57a002d365489ce583eb7f84f, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,079 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2424ff4b6d907aafacef6f661d88ef95, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,079 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51bf57bcf1afa9f625176363284d17f7, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,079 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93deaad3e3229244196e8daa9478371a, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=5 2024-11-15T22:14:42,080 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ecaeddccb9b2987f43bb050f0a105976, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,080 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1959cb95bb21ecdc677f7f5290173984, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,080 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13c3457b9645c9a8181ed9b4551962ad, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,080 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1732b30f3d981289760d152f788aff6f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,080 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36b8b3bed3adb94f5901f1f3809b64fa, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,080 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2a159d388ea2138ee8e2bed061a785ce, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,080 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de9977b187fcaabda585b015a568e905, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,080 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f64d8a749084ba25ea1b77db9a0d9f17, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,080 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c56d152c07a95a189ac65341ff4db1fa, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,080 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a602f0e8bd42c685584b8b985174ddea, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,081 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83048e0be415b190a19f64bc2e49d3fe, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,081 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06b9df8f55d791077e9f7ec55ef6a971, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,081 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7502fc847b4b97e1ace9a22ec1c068e0, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,081 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=819421a6406c0ebb5635c399b2c0303d, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,081 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4381d14830a288bee3aeb42195dd8768, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,081 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e87a82b283b7251342a664ba3876c98e, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,081 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9e6155d6c82d9d9a5c9449be6d7ce5f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,081 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e79a8ce6f9715045d056d3dfdef5cec, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,081 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7fb991e3993a690fe7a0a1b064d5f5d, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,081 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1cecce90107a4c37b7d91651975993f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,082 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6efdb15046141ab0cd574540ef2ac2c2, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,082 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-15T22:14:42,082 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731708942082 2024-11-15T22:14:42,082 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731709002082 2024-11-15T22:14:42,082 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 7 msec 2024-11-15T22:14:42,082 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T22:14:42,082 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T22:14:42,082 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T22:14:42,082 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T22:14:42,082 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T22:14:42,082 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T22:14:42,082 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T22:14:42,083 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T22:14:42,083 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-15T22:14:42,085 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-15T22:14:42,085 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T22:14:42,085 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T22:14:42,085 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:42,085 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:42,085 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:42,085 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:42,085 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:42,085 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T22:14:42,085 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T22:14:42,085 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:42,085 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T22:14:42,085 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T22:14:42,086 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-15T22:14:42,086 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=93deaad3e3229244196e8daa9478371a, REOPEN/MOVE 2024-11-15T22:14:42,086 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true 2024-11-15T22:14:42,086 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:42,086 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:42,087 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T22:14:42,087 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e611192d6313,38729,1731708868111 2024-11-15T22:14:42,087 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:42,087 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:42,087 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN 2024-11-15T22:14:42,087 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN 2024-11-15T22:14:42,087 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN 2024-11-15T22:14:42,087 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f2783381a48c13c2f75c07e5029903cb, ASSIGN 2024-11-15T22:14:42,087 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06b9df8f55d791077e9f7ec55ef6a971, ASSIGN 2024-11-15T22:14:42,087 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:42,087 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,087 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,087 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,087 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e611192d6313,38729,1731708868111 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,087 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,087 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,087 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:42,087 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN 2024-11-15T22:14:42,087 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:42,087 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN 2024-11-15T22:14:42,087 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:42,087 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN 2024-11-15T22:14:42,087 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f2783381a48c13c2f75c07e5029903cb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:42,087 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f2783381a48c13c2f75c07e5029903cb, ASSIGN 2024-11-15T22:14:42,087 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06b9df8f55d791077e9f7ec55ef6a971, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:42,087 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06b9df8f55d791077e9f7ec55ef6a971, ASSIGN 2024-11-15T22:14:42,088 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-15T22:14:42,088 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=65 2024-11-15T22:14:42,088 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=e611192d6313,38729,1731708868111, table=hbase:meta, region=1588230740 2024-11-15T22:14:42,089 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e611192d6313,38729,1731708868111, state=OPEN 2024-11-15T22:14:42,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39087-0x101409049d30000, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:42,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x101409049d30001, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:42,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41231-0x101409049d30002, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:42,099 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:42,099 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:42,099 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:42,100 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN to state=OFFLINE, location=null, table=test, region=819421a6406c0ebb5635c399b2c0303d to restore RIT 2024-11-15T22:14:42,100 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN to state=OFFLINE, location=null, table=test, region=2a159d388ea2138ee8e2bed061a785ce to restore RIT 2024-11-15T22:14:42,100 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN to state=OFFLINE, location=null, table=test, region=13c3457b9645c9a8181ed9b4551962ad to restore RIT 2024-11-15T22:14:42,100 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f2783381a48c13c2f75c07e5029903cb, ASSIGN to state=OFFLINE, location=null, table=test, region=f2783381a48c13c2f75c07e5029903cb to restore RIT 2024-11-15T22:14:42,100 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06b9df8f55d791077e9f7ec55ef6a971, ASSIGN to state=OFFLINE, location=null, table=test, region=06b9df8f55d791077e9f7ec55ef6a971 to restore RIT 2024-11-15T22:14:42,100 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731708912100 2024-11-15T22:14:42,100 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-15T22:14:42,100 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:14:42,101 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-15T22:14:42,102 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-15T22:14:42,102 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f2783381a48c13c2f75c07e5029903cb, ASSIGN 2024-11-15T22:14:42,102 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T22:14:42,102 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f2783381a48c13c2f75c07e5029903cb, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:14:42,104 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f2783381a48c13c2f75c07e5029903cb, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,105 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=897a7bb57a002d365489ce583eb7f84f, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,105 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2424ff4b6d907aafacef6f661d88ef95, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,105 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51bf57bcf1afa9f625176363284d17f7, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,105 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93deaad3e3229244196e8daa9478371a, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=5 2024-11-15T22:14:42,105 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ecaeddccb9b2987f43bb050f0a105976, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,105 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1959cb95bb21ecdc677f7f5290173984, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,105 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13c3457b9645c9a8181ed9b4551962ad, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,105 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1732b30f3d981289760d152f788aff6f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,105 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36b8b3bed3adb94f5901f1f3809b64fa, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,105 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2a159d388ea2138ee8e2bed061a785ce, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,106 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de9977b187fcaabda585b015a568e905, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,106 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f64d8a749084ba25ea1b77db9a0d9f17, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,106 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c56d152c07a95a189ac65341ff4db1fa, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,106 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a602f0e8bd42c685584b8b985174ddea, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,106 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83048e0be415b190a19f64bc2e49d3fe, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,106 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06b9df8f55d791077e9f7ec55ef6a971, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,106 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7502fc847b4b97e1ace9a22ec1c068e0, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,106 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=819421a6406c0ebb5635c399b2c0303d, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,106 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4381d14830a288bee3aeb42195dd8768, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,106 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e87a82b283b7251342a664ba3876c98e, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,107 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9e6155d6c82d9d9a5c9449be6d7ce5f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,107 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e79a8ce6f9715045d056d3dfdef5cec, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,107 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7fb991e3993a690fe7a0a1b064d5f5d, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,107 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1cecce90107a4c37b7d91651975993f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,107 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6efdb15046141ab0cd574540ef2ac2c2, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,107 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-15T22:14:42,107 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731708942107 2024-11-15T22:14:42,107 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731709002107 2024-11-15T22:14:42,107 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 7 msec 2024-11-15T22:14:42,107 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T22:14:42,107 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T22:14:42,107 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T22:14:42,107 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T22:14:42,108 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T22:14:42,108 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T22:14:42,108 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T22:14:42,108 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T22:14:42,109 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-15T22:14:42,110 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-15T22:14:42,110 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T22:14:42,110 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T22:14:42,110 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:42,110 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:42,110 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:42,110 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:42,110 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T22:14:42,110 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T22:14:42,110 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:42,110 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T22:14:42,111 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T22:14:42,111 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-15T22:14:42,111 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=93deaad3e3229244196e8daa9478371a, REOPEN/MOVE 2024-11-15T22:14:42,112 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true 2024-11-15T22:14:42,112 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:42,112 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:42,112 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T22:14:42,112 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e611192d6313,38729,1731708868111 2024-11-15T22:14:42,112 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:42,112 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:42,112 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN 2024-11-15T22:14:42,112 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN 2024-11-15T22:14:42,112 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN 2024-11-15T22:14:42,112 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f2783381a48c13c2f75c07e5029903cb, ASSIGN 2024-11-15T22:14:42,112 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:42,112 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,112 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,112 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,112 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e611192d6313,38729,1731708868111 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,112 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,113 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,113 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:42,113 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN 2024-11-15T22:14:42,113 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:42,113 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN 2024-11-15T22:14:42,113 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:42,113 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN 2024-11-15T22:14:42,113 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f2783381a48c13c2f75c07e5029903cb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:42,113 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f2783381a48c13c2f75c07e5029903cb, ASSIGN 2024-11-15T22:14:42,113 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-15T22:14:42,114 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=65 2024-11-15T22:14:42,114 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=e611192d6313,38729,1731708868111, table=hbase:meta, region=1588230740 2024-11-15T22:14:42,114 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e611192d6313,38729,1731708868111, state=OPEN 2024-11-15T22:14:42,126 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39087-0x101409049d30000, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:42,126 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x101409049d30001, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:42,126 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41231-0x101409049d30002, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:42,126 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:42,126 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:42,126 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:42,127 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN to state=OFFLINE, location=null, table=test, region=819421a6406c0ebb5635c399b2c0303d to restore RIT 2024-11-15T22:14:42,127 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN to state=OFFLINE, location=null, table=test, region=2a159d388ea2138ee8e2bed061a785ce to restore RIT 2024-11-15T22:14:42,127 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN to state=OFFLINE, location=null, table=test, region=13c3457b9645c9a8181ed9b4551962ad to restore RIT 2024-11-15T22:14:42,127 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f2783381a48c13c2f75c07e5029903cb, ASSIGN to state=OFFLINE, location=null, table=test, region=f2783381a48c13c2f75c07e5029903cb to restore RIT 2024-11-15T22:14:42,127 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731708912127 2024-11-15T22:14:42,127 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-15T22:14:42,127 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:14:42,128 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-15T22:14:42,129 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-15T22:14:42,129 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN 2024-11-15T22:14:42,129 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T22:14:42,129 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:14:42,132 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f2783381a48c13c2f75c07e5029903cb, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,132 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=897a7bb57a002d365489ce583eb7f84f, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,132 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2424ff4b6d907aafacef6f661d88ef95, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,132 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51bf57bcf1afa9f625176363284d17f7, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,132 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93deaad3e3229244196e8daa9478371a, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=5 2024-11-15T22:14:42,132 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ecaeddccb9b2987f43bb050f0a105976, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,132 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1959cb95bb21ecdc677f7f5290173984, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,132 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13c3457b9645c9a8181ed9b4551962ad, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,132 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1732b30f3d981289760d152f788aff6f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,133 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36b8b3bed3adb94f5901f1f3809b64fa, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,133 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2a159d388ea2138ee8e2bed061a785ce, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,133 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de9977b187fcaabda585b015a568e905, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,133 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f64d8a749084ba25ea1b77db9a0d9f17, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,133 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c56d152c07a95a189ac65341ff4db1fa, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,133 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a602f0e8bd42c685584b8b985174ddea, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,133 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83048e0be415b190a19f64bc2e49d3fe, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,133 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06b9df8f55d791077e9f7ec55ef6a971, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,133 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7502fc847b4b97e1ace9a22ec1c068e0, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,133 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=819421a6406c0ebb5635c399b2c0303d, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,133 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4381d14830a288bee3aeb42195dd8768, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,134 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e87a82b283b7251342a664ba3876c98e, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,134 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9e6155d6c82d9d9a5c9449be6d7ce5f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,134 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e79a8ce6f9715045d056d3dfdef5cec, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,134 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7fb991e3993a690fe7a0a1b064d5f5d, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,134 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1cecce90107a4c37b7d91651975993f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,134 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6efdb15046141ab0cd574540ef2ac2c2, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,134 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-15T22:14:42,134 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731708942134 2024-11-15T22:14:42,134 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731709002134 2024-11-15T22:14:42,134 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 6 msec 2024-11-15T22:14:42,134 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T22:14:42,134 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T22:14:42,134 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T22:14:42,134 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T22:14:42,135 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T22:14:42,135 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T22:14:42,135 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T22:14:42,135 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T22:14:42,136 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-15T22:14:42,137 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-15T22:14:42,137 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T22:14:42,137 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T22:14:42,137 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:42,137 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:42,137 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:42,137 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T22:14:42,137 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T22:14:42,137 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:42,137 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T22:14:42,137 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T22:14:42,138 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-15T22:14:42,138 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=93deaad3e3229244196e8daa9478371a, REOPEN/MOVE 2024-11-15T22:14:42,138 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true 2024-11-15T22:14:42,139 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:42,139 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:42,139 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T22:14:42,139 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e611192d6313,38729,1731708868111 2024-11-15T22:14:42,139 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:42,139 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:42,139 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN 2024-11-15T22:14:42,139 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN 2024-11-15T22:14:42,139 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN 2024-11-15T22:14:42,139 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:42,139 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,139 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,139 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,139 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e611192d6313,38729,1731708868111 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,139 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,139 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,139 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:42,139 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN 2024-11-15T22:14:42,139 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:42,139 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN 2024-11-15T22:14:42,139 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:42,139 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN 2024-11-15T22:14:42,139 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-15T22:14:42,140 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=65 2024-11-15T22:14:42,140 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=e611192d6313,38729,1731708868111, table=hbase:meta, region=1588230740 2024-11-15T22:14:42,140 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e611192d6313,38729,1731708868111, state=OPEN 2024-11-15T22:14:42,148 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39087 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:14:42,148 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39087 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:14:42,151 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x101409049d30001, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:42,151 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41231-0x101409049d30002, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:42,151 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39087-0x101409049d30000, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:42,152 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:42,152 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:42,152 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:42,152 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN to state=OFFLINE, location=null, table=test, region=819421a6406c0ebb5635c399b2c0303d to restore RIT 2024-11-15T22:14:42,152 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN to state=OFFLINE, location=null, table=test, region=2a159d388ea2138ee8e2bed061a785ce to restore RIT 2024-11-15T22:14:42,152 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=13c3457b9645c9a8181ed9b4551962ad, ASSIGN to state=OFFLINE, location=null, table=test, region=13c3457b9645c9a8181ed9b4551962ad to restore RIT 2024-11-15T22:14:42,153 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731708912153 2024-11-15T22:14:42,153 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-15T22:14:42,153 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:14:42,154 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-15T22:14:42,155 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-15T22:14:42,155 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN 2024-11-15T22:14:42,155 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T22:14:42,155 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:14:42,157 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f2783381a48c13c2f75c07e5029903cb, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,157 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=897a7bb57a002d365489ce583eb7f84f, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,157 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2424ff4b6d907aafacef6f661d88ef95, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,157 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51bf57bcf1afa9f625176363284d17f7, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,157 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93deaad3e3229244196e8daa9478371a, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=5 2024-11-15T22:14:42,157 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ecaeddccb9b2987f43bb050f0a105976, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,157 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1959cb95bb21ecdc677f7f5290173984, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,158 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13c3457b9645c9a8181ed9b4551962ad, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,158 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1732b30f3d981289760d152f788aff6f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,158 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36b8b3bed3adb94f5901f1f3809b64fa, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,158 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2a159d388ea2138ee8e2bed061a785ce, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,158 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de9977b187fcaabda585b015a568e905, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,158 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f64d8a749084ba25ea1b77db9a0d9f17, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,158 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c56d152c07a95a189ac65341ff4db1fa, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,158 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a602f0e8bd42c685584b8b985174ddea, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,158 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83048e0be415b190a19f64bc2e49d3fe, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,158 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06b9df8f55d791077e9f7ec55ef6a971, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,158 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7502fc847b4b97e1ace9a22ec1c068e0, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=819421a6406c0ebb5635c399b2c0303d, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4381d14830a288bee3aeb42195dd8768, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e87a82b283b7251342a664ba3876c98e, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9e6155d6c82d9d9a5c9449be6d7ce5f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e79a8ce6f9715045d056d3dfdef5cec, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7fb991e3993a690fe7a0a1b064d5f5d, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1cecce90107a4c37b7d91651975993f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6efdb15046141ab0cd574540ef2ac2c2, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,159 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-15T22:14:42,159 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731708942159 2024-11-15T22:14:42,159 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731709002159 2024-11-15T22:14:42,159 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 6 msec 2024-11-15T22:14:42,159 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T22:14:42,159 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T22:14:42,159 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T22:14:42,160 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T22:14:42,160 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T22:14:42,160 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T22:14:42,160 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T22:14:42,160 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T22:14:42,161 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-15T22:14:42,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-15T22:14:42,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T22:14:42,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T22:14:42,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:42,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:42,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T22:14:42,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T22:14:42,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:42,162 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T22:14:42,163 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T22:14:42,163 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-15T22:14:42,163 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=93deaad3e3229244196e8daa9478371a, REOPEN/MOVE 2024-11-15T22:14:42,164 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true 2024-11-15T22:14:42,164 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:42,164 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:42,164 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T22:14:42,164 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e611192d6313,38729,1731708868111 2024-11-15T22:14:42,164 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:42,164 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:42,164 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN 2024-11-15T22:14:42,164 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN 2024-11-15T22:14:42,164 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:42,164 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,164 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,164 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,164 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e611192d6313,38729,1731708868111 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,164 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,164 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,164 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:42,165 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN 2024-11-15T22:14:42,165 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:42,165 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN 2024-11-15T22:14:42,165 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-15T22:14:42,165 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=65 2024-11-15T22:14:42,165 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=e611192d6313,38729,1731708868111, table=hbase:meta, region=1588230740 2024-11-15T22:14:42,166 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e611192d6313,38729,1731708868111, state=OPEN 2024-11-15T22:14:42,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39087-0x101409049d30000, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:42,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41231-0x101409049d30002, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:42,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x101409049d30001, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:42,208 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:42,208 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:42,208 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:42,208 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN to state=OFFLINE, location=null, table=test, region=819421a6406c0ebb5635c399b2c0303d to restore RIT 2024-11-15T22:14:42,209 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2a159d388ea2138ee8e2bed061a785ce, ASSIGN to state=OFFLINE, location=null, table=test, region=2a159d388ea2138ee8e2bed061a785ce to restore RIT 2024-11-15T22:14:42,209 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731708912209 2024-11-15T22:14:42,209 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-15T22:14:42,209 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:14:42,210 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-15T22:14:42,211 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-15T22:14:42,211 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN 2024-11-15T22:14:42,211 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T22:14:42,211 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:14:42,215 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f2783381a48c13c2f75c07e5029903cb, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,215 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=897a7bb57a002d365489ce583eb7f84f, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,215 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2424ff4b6d907aafacef6f661d88ef95, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,215 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51bf57bcf1afa9f625176363284d17f7, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,215 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93deaad3e3229244196e8daa9478371a, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=5 2024-11-15T22:14:42,216 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ecaeddccb9b2987f43bb050f0a105976, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,216 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1959cb95bb21ecdc677f7f5290173984, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,216 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13c3457b9645c9a8181ed9b4551962ad, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,216 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1732b30f3d981289760d152f788aff6f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,216 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36b8b3bed3adb94f5901f1f3809b64fa, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,216 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2a159d388ea2138ee8e2bed061a785ce, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,217 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de9977b187fcaabda585b015a568e905, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,217 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f64d8a749084ba25ea1b77db9a0d9f17, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,217 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c56d152c07a95a189ac65341ff4db1fa, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,217 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a602f0e8bd42c685584b8b985174ddea, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,217 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83048e0be415b190a19f64bc2e49d3fe, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,217 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06b9df8f55d791077e9f7ec55ef6a971, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,217 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7502fc847b4b97e1ace9a22ec1c068e0, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,218 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=819421a6406c0ebb5635c399b2c0303d, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,218 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4381d14830a288bee3aeb42195dd8768, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,218 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e87a82b283b7251342a664ba3876c98e, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,218 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9e6155d6c82d9d9a5c9449be6d7ce5f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,218 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e79a8ce6f9715045d056d3dfdef5cec, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,218 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7fb991e3993a690fe7a0a1b064d5f5d, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,219 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1cecce90107a4c37b7d91651975993f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,219 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6efdb15046141ab0cd574540ef2ac2c2, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,219 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-15T22:14:42,219 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731708942219 2024-11-15T22:14:42,219 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731709002219 2024-11-15T22:14:42,219 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-15T22:14:42,219 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T22:14:42,219 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T22:14:42,219 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T22:14:42,219 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T22:14:42,220 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T22:14:42,220 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T22:14:42,220 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T22:14:42,220 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T22:14:42,221 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-15T22:14:42,223 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-15T22:14:42,223 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T22:14:42,223 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T22:14:42,224 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:42,224 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T22:14:42,224 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T22:14:42,224 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:42,224 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T22:14:42,224 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T22:14:42,225 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-15T22:14:42,225 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=93deaad3e3229244196e8daa9478371a, REOPEN/MOVE 2024-11-15T22:14:42,226 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true 2024-11-15T22:14:42,226 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:42,226 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:42,226 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T22:14:42,226 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e611192d6313,38729,1731708868111 2024-11-15T22:14:42,226 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:42,226 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:42,227 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN 2024-11-15T22:14:42,227 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:42,227 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,227 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,227 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,227 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e611192d6313,38729,1731708868111 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,227 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,227 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,227 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:42,227 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN 2024-11-15T22:14:42,227 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-15T22:14:42,228 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=65 2024-11-15T22:14:42,228 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=e611192d6313,38729,1731708868111, table=hbase:meta, region=1588230740 2024-11-15T22:14:42,228 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e611192d6313,38729,1731708868111, state=OPEN 2024-11-15T22:14:42,246 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41231-0x101409049d30002, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:42,246 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x101409049d30001, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:42,246 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39087-0x101409049d30000, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:42,246 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:42,246 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:42,246 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:42,247 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=819421a6406c0ebb5635c399b2c0303d, ASSIGN to state=OFFLINE, location=null, table=test, region=819421a6406c0ebb5635c399b2c0303d to restore RIT 2024-11-15T22:14:42,247 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731708912247 2024-11-15T22:14:42,247 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-15T22:14:42,247 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:14:42,249 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-15T22:14:42,249 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-15T22:14:42,249 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:42,249 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T22:14:42,249 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:14:42,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f2783381a48c13c2f75c07e5029903cb, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=897a7bb57a002d365489ce583eb7f84f, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2424ff4b6d907aafacef6f661d88ef95, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51bf57bcf1afa9f625176363284d17f7, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93deaad3e3229244196e8daa9478371a, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=5 2024-11-15T22:14:42,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ecaeddccb9b2987f43bb050f0a105976, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1959cb95bb21ecdc677f7f5290173984, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13c3457b9645c9a8181ed9b4551962ad, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1732b30f3d981289760d152f788aff6f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36b8b3bed3adb94f5901f1f3809b64fa, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2a159d388ea2138ee8e2bed061a785ce, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de9977b187fcaabda585b015a568e905, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f64d8a749084ba25ea1b77db9a0d9f17, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c56d152c07a95a189ac65341ff4db1fa, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a602f0e8bd42c685584b8b985174ddea, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83048e0be415b190a19f64bc2e49d3fe, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06b9df8f55d791077e9f7ec55ef6a971, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7502fc847b4b97e1ace9a22ec1c068e0, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=819421a6406c0ebb5635c399b2c0303d, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4381d14830a288bee3aeb42195dd8768, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e87a82b283b7251342a664ba3876c98e, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9e6155d6c82d9d9a5c9449be6d7ce5f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e79a8ce6f9715045d056d3dfdef5cec, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7fb991e3993a690fe7a0a1b064d5f5d, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1cecce90107a4c37b7d91651975993f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6efdb15046141ab0cd574540ef2ac2c2, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,254 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-15T22:14:42,254 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731708942254 2024-11-15T22:14:42,254 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731709002254 2024-11-15T22:14:42,254 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 7 msec 2024-11-15T22:14:42,254 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T22:14:42,254 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T22:14:42,254 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T22:14:42,255 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T22:14:42,255 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T22:14:42,255 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T22:14:42,255 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T22:14:42,255 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T22:14:42,256 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-15T22:14:42,257 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-15T22:14:42,257 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T22:14:42,257 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T22:14:42,257 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T22:14:42,257 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T22:14:42,257 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:42,257 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T22:14:42,257 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T22:14:42,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-15T22:14:42,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=93deaad3e3229244196e8daa9478371a, REOPEN/MOVE 2024-11-15T22:14:42,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true 2024-11-15T22:14:42,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:42,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:42,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T22:14:42,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e611192d6313,38729,1731708868111 2024-11-15T22:14:42,259 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:42,259 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:42,259 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:42,259 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,259 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,259 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,259 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e611192d6313,38729,1731708868111 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,259 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,259 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,259 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-15T22:14:42,259 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=65 2024-11-15T22:14:42,260 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=e611192d6313,38729,1731708868111, table=hbase:meta, region=1588230740 2024-11-15T22:14:42,260 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e611192d6313,38729,1731708868111, state=OPEN 2024-11-15T22:14:42,267 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x101409049d30001, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:42,267 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41231-0x101409049d30002, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:42,267 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39087-0x101409049d30000, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:42,268 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:42,268 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:42,268 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:42,269 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731708912269 2024-11-15T22:14:42,269 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-15T22:14:42,269 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:14:42,270 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-15T22:14:42,271 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-15T22:14:42,271 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:42,271 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T22:14:42,271 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:14:42,273 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f2783381a48c13c2f75c07e5029903cb, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,273 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=897a7bb57a002d365489ce583eb7f84f, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,273 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2424ff4b6d907aafacef6f661d88ef95, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,273 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51bf57bcf1afa9f625176363284d17f7, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,273 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93deaad3e3229244196e8daa9478371a, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=5 2024-11-15T22:14:42,274 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ecaeddccb9b2987f43bb050f0a105976, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,274 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1959cb95bb21ecdc677f7f5290173984, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,274 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13c3457b9645c9a8181ed9b4551962ad, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,274 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1732b30f3d981289760d152f788aff6f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,274 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36b8b3bed3adb94f5901f1f3809b64fa, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,274 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2a159d388ea2138ee8e2bed061a785ce, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,274 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de9977b187fcaabda585b015a568e905, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,274 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f64d8a749084ba25ea1b77db9a0d9f17, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,274 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c56d152c07a95a189ac65341ff4db1fa, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,274 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a602f0e8bd42c685584b8b985174ddea, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,274 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83048e0be415b190a19f64bc2e49d3fe, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,274 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06b9df8f55d791077e9f7ec55ef6a971, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,274 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7502fc847b4b97e1ace9a22ec1c068e0, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,275 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=819421a6406c0ebb5635c399b2c0303d, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,275 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4381d14830a288bee3aeb42195dd8768, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,275 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e87a82b283b7251342a664ba3876c98e, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,275 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9e6155d6c82d9d9a5c9449be6d7ce5f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,275 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e79a8ce6f9715045d056d3dfdef5cec, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,275 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7fb991e3993a690fe7a0a1b064d5f5d, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,275 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1cecce90107a4c37b7d91651975993f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,275 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6efdb15046141ab0cd574540ef2ac2c2, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,275 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-15T22:14:42,275 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731708942275 2024-11-15T22:14:42,275 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731709002275 2024-11-15T22:14:42,275 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 6 msec 2024-11-15T22:14:42,275 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T22:14:42,275 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T22:14:42,275 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T22:14:42,275 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T22:14:42,276 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T22:14:42,276 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T22:14:42,276 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T22:14:42,276 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T22:14:42,277 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-15T22:14:42,278 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-15T22:14:42,278 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T22:14:42,278 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T22:14:42,278 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T22:14:42,278 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:42,278 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T22:14:42,278 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T22:14:42,279 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-15T22:14:42,279 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=93deaad3e3229244196e8daa9478371a, REOPEN/MOVE 2024-11-15T22:14:42,279 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true 2024-11-15T22:14:42,279 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:42,279 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:42,279 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T22:14:42,279 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e611192d6313,38729,1731708868111 2024-11-15T22:14:42,279 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:42,279 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:42,280 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,280 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,280 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,280 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e611192d6313,38729,1731708868111 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,280 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.1731708872733, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,280 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-15T22:14:42,280 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=65 2024-11-15T22:14:42,281 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=e611192d6313,38729,1731708868111, table=hbase:meta, region=1588230740 2024-11-15T22:14:42,281 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e611192d6313,38729,1731708868111, state=OPEN 2024-11-15T22:14:42,288 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x101409049d30001, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:42,288 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39087-0x101409049d30000, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:42,288 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41231-0x101409049d30002, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:42,289 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:42,289 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:42,289 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:42,289 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731708912289 2024-11-15T22:14:42,290 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-15T22:14:42,290 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:14:42,291 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-15T22:14:42,292 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-15T22:14:42,292 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e611192d6313,38729,1731708868111 2024-11-15T22:14:42,292 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T22:14:42,292 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e611192d6313,38729,1731708868111 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:14:42,294 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f2783381a48c13c2f75c07e5029903cb, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,294 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=897a7bb57a002d365489ce583eb7f84f, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,294 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2424ff4b6d907aafacef6f661d88ef95, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,294 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51bf57bcf1afa9f625176363284d17f7, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,294 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93deaad3e3229244196e8daa9478371a, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=5 2024-11-15T22:14:42,294 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ecaeddccb9b2987f43bb050f0a105976, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,294 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1959cb95bb21ecdc677f7f5290173984, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,294 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13c3457b9645c9a8181ed9b4551962ad, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,295 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1732b30f3d981289760d152f788aff6f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,295 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36b8b3bed3adb94f5901f1f3809b64fa, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,295 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2a159d388ea2138ee8e2bed061a785ce, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,295 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de9977b187fcaabda585b015a568e905, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,295 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f64d8a749084ba25ea1b77db9a0d9f17, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,295 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c56d152c07a95a189ac65341ff4db1fa, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,295 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a602f0e8bd42c685584b8b985174ddea, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,295 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83048e0be415b190a19f64bc2e49d3fe, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,295 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06b9df8f55d791077e9f7ec55ef6a971, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,295 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7502fc847b4b97e1ace9a22ec1c068e0, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,295 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=819421a6406c0ebb5635c399b2c0303d, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,295 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4381d14830a288bee3aeb42195dd8768, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,295 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e87a82b283b7251342a664ba3876c98e, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,296 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9e6155d6c82d9d9a5c9449be6d7ce5f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,296 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e79a8ce6f9715045d056d3dfdef5cec, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,296 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7fb991e3993a690fe7a0a1b064d5f5d, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,296 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1cecce90107a4c37b7d91651975993f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,296 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6efdb15046141ab0cd574540ef2ac2c2, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,296 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-15T22:14:42,296 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731708942296 2024-11-15T22:14:42,296 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731709002296 2024-11-15T22:14:42,296 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 6 msec 2024-11-15T22:14:42,296 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T22:14:42,296 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T22:14:42,296 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T22:14:42,296 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T22:14:42,297 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T22:14:42,297 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T22:14:42,297 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T22:14:42,297 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T22:14:42,298 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-15T22:14:42,299 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-15T22:14:42,299 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T22:14:42,299 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T22:14:42,299 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:42,299 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-15T22:14:42,299 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T22:14:42,300 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-15T22:14:42,300 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=93deaad3e3229244196e8daa9478371a, REOPEN/MOVE 2024-11-15T22:14:42,300 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true 2024-11-15T22:14:42,301 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:42,301 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:42,301 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T22:14:42,301 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e611192d6313,38729,1731708868111 2024-11-15T22:14:42,301 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:42,301 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,301 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,301 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,301 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e611192d6313,38729,1731708868111 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,301 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-15T22:14:42,302 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=65 2024-11-15T22:14:42,302 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=e611192d6313,38729,1731708868111, table=hbase:meta, region=1588230740 2024-11-15T22:14:42,302 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e611192d6313,38729,1731708868111, state=OPEN 2024-11-15T22:14:42,315 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39087-0x101409049d30000, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:42,315 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41231-0x101409049d30002, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:42,315 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x101409049d30001, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:42,316 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:42,316 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:42,316 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:42,316 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731708912316 2024-11-15T22:14:42,316 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-15T22:14:42,316 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:14:42,318 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-15T22:14:42,319 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-15T22:14:42,319 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T22:14:42,319 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T22:14:42,319 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:14:42,320 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f2783381a48c13c2f75c07e5029903cb, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,320 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=897a7bb57a002d365489ce583eb7f84f, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,320 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2424ff4b6d907aafacef6f661d88ef95, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51bf57bcf1afa9f625176363284d17f7, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93deaad3e3229244196e8daa9478371a, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=5 2024-11-15T22:14:42,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ecaeddccb9b2987f43bb050f0a105976, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1959cb95bb21ecdc677f7f5290173984, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13c3457b9645c9a8181ed9b4551962ad, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1732b30f3d981289760d152f788aff6f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36b8b3bed3adb94f5901f1f3809b64fa, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2a159d388ea2138ee8e2bed061a785ce, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de9977b187fcaabda585b015a568e905, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f64d8a749084ba25ea1b77db9a0d9f17, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c56d152c07a95a189ac65341ff4db1fa, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a602f0e8bd42c685584b8b985174ddea, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83048e0be415b190a19f64bc2e49d3fe, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06b9df8f55d791077e9f7ec55ef6a971, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,322 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7502fc847b4b97e1ace9a22ec1c068e0, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,322 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=819421a6406c0ebb5635c399b2c0303d, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,322 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4381d14830a288bee3aeb42195dd8768, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,322 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e87a82b283b7251342a664ba3876c98e, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,322 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9e6155d6c82d9d9a5c9449be6d7ce5f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,322 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e79a8ce6f9715045d056d3dfdef5cec, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,322 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7fb991e3993a690fe7a0a1b064d5f5d, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,322 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1cecce90107a4c37b7d91651975993f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,322 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6efdb15046141ab0cd574540ef2ac2c2, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,322 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-15T22:14:42,322 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731708942322 2024-11-15T22:14:42,322 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731709002322 2024-11-15T22:14:42,322 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 5 msec 2024-11-15T22:14:42,322 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T22:14:42,322 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T22:14:42,322 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T22:14:42,323 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T22:14:42,323 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T22:14:42,323 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T22:14:42,323 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T22:14:42,323 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T22:14:42,324 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-15T22:14:42,325 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-15T22:14:42,325 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T22:14:42,325 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T22:14:42,325 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-15T22:14:42,325 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T22:14:42,326 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-15T22:14:42,326 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=93deaad3e3229244196e8daa9478371a, REOPEN/MOVE 2024-11-15T22:14:42,326 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true 2024-11-15T22:14:42,326 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:42,326 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:42,326 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T22:14:42,326 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:42,326 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,326 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,326 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,327 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-15T22:14:42,327 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=65 2024-11-15T22:14:42,327 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=e611192d6313,38729,1731708868111, table=hbase:meta, region=1588230740 2024-11-15T22:14:42,327 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e611192d6313,38729,1731708868111, state=OPEN 2024-11-15T22:14:42,336 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39087-0x101409049d30000, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:42,336 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x101409049d30001, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:42,336 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41231-0x101409049d30002, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:42,337 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:42,337 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:42,337 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:42,337 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731708912337 2024-11-15T22:14:42,337 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-15T22:14:42,337 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:14:42,338 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-15T22:14:42,339 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-15T22:14:42,339 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:42,339 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T22:14:42,339 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:14:42,341 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f2783381a48c13c2f75c07e5029903cb, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,341 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=897a7bb57a002d365489ce583eb7f84f, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,341 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2424ff4b6d907aafacef6f661d88ef95, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,341 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51bf57bcf1afa9f625176363284d17f7, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,342 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93deaad3e3229244196e8daa9478371a, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=5 2024-11-15T22:14:42,342 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ecaeddccb9b2987f43bb050f0a105976, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,342 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1959cb95bb21ecdc677f7f5290173984, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,342 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13c3457b9645c9a8181ed9b4551962ad, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,342 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1732b30f3d981289760d152f788aff6f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,342 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36b8b3bed3adb94f5901f1f3809b64fa, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,342 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2a159d388ea2138ee8e2bed061a785ce, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,342 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de9977b187fcaabda585b015a568e905, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,342 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f64d8a749084ba25ea1b77db9a0d9f17, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,342 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c56d152c07a95a189ac65341ff4db1fa, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,342 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a602f0e8bd42c685584b8b985174ddea, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,342 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83048e0be415b190a19f64bc2e49d3fe, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,342 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06b9df8f55d791077e9f7ec55ef6a971, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,343 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7502fc847b4b97e1ace9a22ec1c068e0, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,343 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=819421a6406c0ebb5635c399b2c0303d, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,343 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4381d14830a288bee3aeb42195dd8768, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,343 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e87a82b283b7251342a664ba3876c98e, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,343 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9e6155d6c82d9d9a5c9449be6d7ce5f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,343 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e79a8ce6f9715045d056d3dfdef5cec, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,343 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7fb991e3993a690fe7a0a1b064d5f5d, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,343 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1cecce90107a4c37b7d91651975993f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,343 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6efdb15046141ab0cd574540ef2ac2c2, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,343 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-15T22:14:42,343 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731708942343 2024-11-15T22:14:42,343 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731709002343 2024-11-15T22:14:42,343 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 5 msec 2024-11-15T22:14:42,343 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T22:14:42,343 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T22:14:42,343 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T22:14:42,344 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T22:14:42,344 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T22:14:42,344 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T22:14:42,344 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T22:14:42,344 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T22:14:42,345 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-15T22:14:42,346 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-15T22:14:42,346 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T22:14:42,346 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-15T22:14:42,346 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T22:14:42,346 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-15T22:14:42,347 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=93deaad3e3229244196e8daa9478371a, REOPEN/MOVE 2024-11-15T22:14:42,347 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true 2024-11-15T22:14:42,347 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:42,347 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:42,347 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:42,347 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,347 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,347 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-15T22:14:42,348 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=65 2024-11-15T22:14:42,348 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=e611192d6313,38729,1731708868111, table=hbase:meta, region=1588230740 2024-11-15T22:14:42,348 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e611192d6313,38729,1731708868111, state=OPEN 2024-11-15T22:14:42,355 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39087 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:14:42,355 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39087 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:14:42,357 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41231-0x101409049d30002, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:42,357 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x101409049d30001, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:42,357 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39087-0x101409049d30000, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:42,357 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:42,357 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:42,357 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:42,358 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731708912358 2024-11-15T22:14:42,358 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-15T22:14:42,358 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:14:42,360 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-15T22:14:42,361 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-15T22:14:42,361 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:42,361 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T22:14:42,361 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:14:42,363 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f2783381a48c13c2f75c07e5029903cb, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,363 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=897a7bb57a002d365489ce583eb7f84f, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,363 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2424ff4b6d907aafacef6f661d88ef95, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,363 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51bf57bcf1afa9f625176363284d17f7, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,363 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93deaad3e3229244196e8daa9478371a, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=5 2024-11-15T22:14:42,363 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ecaeddccb9b2987f43bb050f0a105976, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,363 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1959cb95bb21ecdc677f7f5290173984, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,363 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13c3457b9645c9a8181ed9b4551962ad, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,363 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1732b30f3d981289760d152f788aff6f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,363 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36b8b3bed3adb94f5901f1f3809b64fa, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,363 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2a159d388ea2138ee8e2bed061a785ce, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,364 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de9977b187fcaabda585b015a568e905, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,364 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f64d8a749084ba25ea1b77db9a0d9f17, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,364 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c56d152c07a95a189ac65341ff4db1fa, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,364 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a602f0e8bd42c685584b8b985174ddea, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,364 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83048e0be415b190a19f64bc2e49d3fe, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,364 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06b9df8f55d791077e9f7ec55ef6a971, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,364 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7502fc847b4b97e1ace9a22ec1c068e0, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,364 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=819421a6406c0ebb5635c399b2c0303d, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,364 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4381d14830a288bee3aeb42195dd8768, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,364 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e87a82b283b7251342a664ba3876c98e, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,364 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9e6155d6c82d9d9a5c9449be6d7ce5f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,364 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e79a8ce6f9715045d056d3dfdef5cec, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,364 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7fb991e3993a690fe7a0a1b064d5f5d, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,365 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1cecce90107a4c37b7d91651975993f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,365 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6efdb15046141ab0cd574540ef2ac2c2, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,365 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-15T22:14:42,365 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731708942365 2024-11-15T22:14:42,365 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731709002365 2024-11-15T22:14:42,365 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 6 msec 2024-11-15T22:14:42,365 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T22:14:42,365 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T22:14:42,365 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T22:14:42,365 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T22:14:42,365 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T22:14:42,365 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T22:14:42,365 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T22:14:42,365 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T22:14:42,366 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-15T22:14:42,368 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-15T22:14:42,368 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-15T22:14:42,368 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T22:14:42,368 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-15T22:14:42,369 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=93deaad3e3229244196e8daa9478371a, REOPEN/MOVE 2024-11-15T22:14:42,369 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true 2024-11-15T22:14:42,369 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 2024-11-15T22:14:42,369 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:42,369 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e611192d6313%2C43349%2C1731708869385.meta.1731708873225.meta, worker=e611192d6313,41231,1731708869041 didn't hold the lock before restarting, skip acquiring lock. 2024-11-15T22:14:42,369 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-15T22:14:42,370 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=65 2024-11-15T22:14:42,370 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=e611192d6313,38729,1731708868111, table=hbase:meta, region=1588230740 2024-11-15T22:14:42,370 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e611192d6313,38729,1731708868111, state=OPEN 2024-11-15T22:14:42,383 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39087-0x101409049d30000, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:42,383 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41231-0x101409049d30002, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:42,383 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x101409049d30001, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:42,383 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:42,383 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:42,383 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:42,384 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731708912384 2024-11-15T22:14:42,384 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-15T22:14:42,384 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:14:42,385 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-15T22:14:42,386 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-15T22:14:42,386 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true 2024-11-15T22:14:42,387 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T22:14:42,387 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1604) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:14:42,388 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f2783381a48c13c2f75c07e5029903cb, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,389 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=897a7bb57a002d365489ce583eb7f84f, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,389 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2424ff4b6d907aafacef6f661d88ef95, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,389 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51bf57bcf1afa9f625176363284d17f7, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,389 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93deaad3e3229244196e8daa9478371a, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=5 2024-11-15T22:14:42,389 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ecaeddccb9b2987f43bb050f0a105976, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,389 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1959cb95bb21ecdc677f7f5290173984, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,389 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13c3457b9645c9a8181ed9b4551962ad, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,390 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1732b30f3d981289760d152f788aff6f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,390 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36b8b3bed3adb94f5901f1f3809b64fa, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,390 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2a159d388ea2138ee8e2bed061a785ce, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,390 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de9977b187fcaabda585b015a568e905, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,390 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f64d8a749084ba25ea1b77db9a0d9f17, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,390 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c56d152c07a95a189ac65341ff4db1fa, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,390 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a602f0e8bd42c685584b8b985174ddea, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,390 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83048e0be415b190a19f64bc2e49d3fe, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,391 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06b9df8f55d791077e9f7ec55ef6a971, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,391 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7502fc847b4b97e1ace9a22ec1c068e0, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,391 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=819421a6406c0ebb5635c399b2c0303d, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,391 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4381d14830a288bee3aeb42195dd8768, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,391 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e87a82b283b7251342a664ba3876c98e, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,391 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9e6155d6c82d9d9a5c9449be6d7ce5f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,391 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e79a8ce6f9715045d056d3dfdef5cec, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,391 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7fb991e3993a690fe7a0a1b064d5f5d, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,392 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1cecce90107a4c37b7d91651975993f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,392 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6efdb15046141ab0cd574540ef2ac2c2, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,392 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-15T22:14:42,392 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731708942392 2024-11-15T22:14:42,392 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731709002392 2024-11-15T22:14:42,392 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 7 msec 2024-11-15T22:14:42,392 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-15T22:14:42,392 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T22:14:42,392 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T22:14:42,392 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T22:14:42,393 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-15T22:14:42,393 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-15T22:14:42,393 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-15T22:14:42,393 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T22:14:42,394 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-15T22:14:42,396 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-15T22:14:42,396 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T22:14:42,396 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-15T22:14:42,396 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=93deaad3e3229244196e8daa9478371a, REOPEN/MOVE 2024-11-15T22:14:42,397 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true 2024-11-15T22:14:42,397 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-15T22:14:42,397 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-15T22:14:42,398 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=65 2024-11-15T22:14:42,398 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=e611192d6313,38729,1731708868111, table=hbase:meta, region=1588230740 2024-11-15T22:14:42,398 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e611192d6313,38729,1731708868111, state=OPEN 2024-11-15T22:14:42,410 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x101409049d30001, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:42,410 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41231-0x101409049d30002, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:42,410 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39087-0x101409049d30000, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T22:14:42,410 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:42,410 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:42,411 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T22:14:42,411 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731708912411 2024-11-15T22:14:42,412 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-15T22:14:42,412 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-15T22:14:42,413 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-15T22:14:42,415 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=60, state=ROLLEDBACK, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure 13c3457b9645c9a8181ed9b4551962ad, server=e611192d6313,38729,1731708868111:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e611192d6313,43349,1731708869385, splitWal=true, meta=true exec-time=2.5380 sec 2024-11-15T22:14:42,417 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f2783381a48c13c2f75c07e5029903cb, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,417 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=897a7bb57a002d365489ce583eb7f84f, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,417 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2424ff4b6d907aafacef6f661d88ef95, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,417 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51bf57bcf1afa9f625176363284d17f7, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,417 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=93deaad3e3229244196e8daa9478371a, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=5 2024-11-15T22:14:42,417 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ecaeddccb9b2987f43bb050f0a105976, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,417 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1959cb95bb21ecdc677f7f5290173984, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,417 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=13c3457b9645c9a8181ed9b4551962ad, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,418 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1732b30f3d981289760d152f788aff6f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,418 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=36b8b3bed3adb94f5901f1f3809b64fa, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,418 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2a159d388ea2138ee8e2bed061a785ce, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,418 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de9977b187fcaabda585b015a568e905, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,418 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f64d8a749084ba25ea1b77db9a0d9f17, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,418 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c56d152c07a95a189ac65341ff4db1fa, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,418 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a602f0e8bd42c685584b8b985174ddea, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,418 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=83048e0be415b190a19f64bc2e49d3fe, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,418 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06b9df8f55d791077e9f7ec55ef6a971, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,418 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7502fc847b4b97e1ace9a22ec1c068e0, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,418 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=819421a6406c0ebb5635c399b2c0303d, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,419 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4381d14830a288bee3aeb42195dd8768, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,419 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e87a82b283b7251342a664ba3876c98e, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,419 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9e6155d6c82d9d9a5c9449be6d7ce5f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,419 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2e79a8ce6f9715045d056d3dfdef5cec, regionState=OPENING, lastHost=e611192d6313,43349,1731708869385, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,419 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7fb991e3993a690fe7a0a1b064d5f5d, regionState=OPEN, lastHost=e611192d6313,41231,1731708869041, regionLocation=e611192d6313,41231,1731708869041, openSeqNum=2 2024-11-15T22:14:42,419 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c1cecce90107a4c37b7d91651975993f, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,419 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6efdb15046141ab0cd574540ef2ac2c2, regionState=OPEN, lastHost=e611192d6313,38729,1731708868111, regionLocation=e611192d6313,38729,1731708868111, openSeqNum=2 2024-11-15T22:14:42,419 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-15T22:14:42,419 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731708942419 2024-11-15T22:14:42,419 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731709002419 2024-11-15T22:14:42,419 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 7 msec 2024-11-15T22:14:42,433 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.assignment.TestRollbackSCP#testFailAndRollback Thread=345 (was 377), OpenFileDescriptor=616 (was 635), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=917 (was 971), ProcessCount=11 (was 11), AvailableMemoryMB=231 (was 265) 2024-11-15T22:14:42,434 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-15T22:14:42,434 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T22:14:42,434 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T22:14:42,434 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:14:42,434 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:14:42,434 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T22:14:42,435 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-15T22:14:42,435 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=2049528699, stopped=false 2024-11-15T22:14:42,435 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=e611192d6313,39087,1731708867142 2024-11-15T22:14:42,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39087-0x101409049d30000, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-15T22:14:42,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41231-0x101409049d30002, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-15T22:14:42,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x101409049d30001, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-15T22:14:42,446 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T22:14:42,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41231-0x101409049d30002, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T22:14:42,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x101409049d30001, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T22:14:42,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39087-0x101409049d30000, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T22:14:42,446 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T22:14:42,447 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T22:14:42,447 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:14:42,447 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38729-0x101409049d30001, quorum=127.0.0.1:60621, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T22:14:42,447 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39087-0x101409049d30000, quorum=127.0.0.1:60621, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T22:14:42,447 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'e611192d6313,38729,1731708868111' ***** 2024-11-15T22:14:42,447 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-15T22:14:42,447 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'e611192d6313,41231,1731708869041' ***** 2024-11-15T22:14:42,447 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41231-0x101409049d30002, quorum=127.0.0.1:60621, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T22:14:42,447 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-15T22:14:42,447 INFO [RS:0;e611192d6313:38729 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-15T22:14:42,448 INFO [RS:1;e611192d6313:41231 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-15T22:14:42,448 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-15T22:14:42,448 INFO [RS:1;e611192d6313:41231 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-15T22:14:42,448 INFO [RS:1;e611192d6313:41231 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-15T22:14:42,448 INFO [RS:0;e611192d6313:38729 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-15T22:14:42,448 INFO [RS:0;e611192d6313:38729 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-15T22:14:42,448 INFO [RS:1;e611192d6313:41231 {}] regionserver.HRegionServer(3091): Received CLOSE for 36b8b3bed3adb94f5901f1f3809b64fa 2024-11-15T22:14:42,448 INFO [RS:0;e611192d6313:38729 {}] regionserver.HRegionServer(3091): Received CLOSE for 2424ff4b6d907aafacef6f661d88ef95 2024-11-15T22:14:42,448 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-15T22:14:42,448 INFO [RS:1;e611192d6313:41231 {}] regionserver.HRegionServer(3091): Received CLOSE for a7fb991e3993a690fe7a0a1b064d5f5d 2024-11-15T22:14:42,448 INFO [RS:1;e611192d6313:41231 {}] regionserver.HRegionServer(3091): Received CLOSE for f64d8a749084ba25ea1b77db9a0d9f17 2024-11-15T22:14:42,448 INFO [RS:1;e611192d6313:41231 {}] regionserver.HRegionServer(3091): Received CLOSE for 1959cb95bb21ecdc677f7f5290173984 2024-11-15T22:14:42,448 INFO [RS:1;e611192d6313:41231 {}] regionserver.HRegionServer(3091): Received CLOSE for 83048e0be415b190a19f64bc2e49d3fe 2024-11-15T22:14:42,449 INFO [RS:1;e611192d6313:41231 {}] regionserver.HRegionServer(3091): Received CLOSE for e87a82b283b7251342a664ba3876c98e 2024-11-15T22:14:42,449 INFO [RS:1;e611192d6313:41231 {}] regionserver.HRegionServer(3091): Received CLOSE for 4381d14830a288bee3aeb42195dd8768 2024-11-15T22:14:42,449 INFO [RS:1;e611192d6313:41231 {}] regionserver.HRegionServer(3091): Received CLOSE for 93deaad3e3229244196e8daa9478371a 2024-11-15T22:14:42,449 INFO [RS:1;e611192d6313:41231 {}] regionserver.HRegionServer(3091): Received CLOSE for 51bf57bcf1afa9f625176363284d17f7 2024-11-15T22:14:42,449 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 36b8b3bed3adb94f5901f1f3809b64fa, disabling compactions & flushes 2024-11-15T22:14:42,449 INFO [RS:1;e611192d6313:41231 {}] regionserver.HRegionServer(959): stopping server e611192d6313,41231,1731708869041 2024-11-15T22:14:42,449 INFO [RS:1;e611192d6313:41231 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-15T22:14:42,449 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,jjj,1731708873681.36b8b3bed3adb94f5901f1f3809b64fa. 2024-11-15T22:14:42,449 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1731708873681.36b8b3bed3adb94f5901f1f3809b64fa. 2024-11-15T22:14:42,449 INFO [RS:1;e611192d6313:41231 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;e611192d6313:41231. 2024-11-15T22:14:42,449 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1731708873681.36b8b3bed3adb94f5901f1f3809b64fa. after waiting 0 ms 2024-11-15T22:14:42,449 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1731708873681.36b8b3bed3adb94f5901f1f3809b64fa. 2024-11-15T22:14:42,449 DEBUG [RS:1;e611192d6313:41231 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T22:14:42,449 DEBUG [RS:1;e611192d6313:41231 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:14:42,449 INFO [RS:0;e611192d6313:38729 {}] regionserver.HRegionServer(3091): Received CLOSE for c56d152c07a95a189ac65341ff4db1fa 2024-11-15T22:14:42,449 INFO [RS:0;e611192d6313:38729 {}] regionserver.HRegionServer(3091): Received CLOSE for f2783381a48c13c2f75c07e5029903cb 2024-11-15T22:14:42,449 INFO [RS:1;e611192d6313:41231 {}] regionserver.HRegionServer(1321): Waiting on 9 regions to close 2024-11-15T22:14:42,449 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2424ff4b6d907aafacef6f661d88ef95, disabling compactions & flushes 2024-11-15T22:14:42,449 INFO [RS:0;e611192d6313:38729 {}] regionserver.HRegionServer(3091): Received CLOSE for 7502fc847b4b97e1ace9a22ec1c068e0 2024-11-15T22:14:42,449 DEBUG [RS:1;e611192d6313:41231 {}] regionserver.HRegionServer(1325): Online Regions={36b8b3bed3adb94f5901f1f3809b64fa=test,jjj,1731708873681.36b8b3bed3adb94f5901f1f3809b64fa., a7fb991e3993a690fe7a0a1b064d5f5d=test,xxx,1731708873681.a7fb991e3993a690fe7a0a1b064d5f5d., f64d8a749084ba25ea1b77db9a0d9f17=test,mmm,1731708873681.f64d8a749084ba25ea1b77db9a0d9f17., 1959cb95bb21ecdc677f7f5290173984=test,ggg,1731708873681.1959cb95bb21ecdc677f7f5290173984., 83048e0be415b190a19f64bc2e49d3fe=test,ppp,1731708873681.83048e0be415b190a19f64bc2e49d3fe., e87a82b283b7251342a664ba3876c98e=test,uuu,1731708873681.e87a82b283b7251342a664ba3876c98e., 4381d14830a288bee3aeb42195dd8768=test,ttt,1731708873681.4381d14830a288bee3aeb42195dd8768., 93deaad3e3229244196e8daa9478371a=test,eee,1731708873681.93deaad3e3229244196e8daa9478371a., 51bf57bcf1afa9f625176363284d17f7=test,ddd,1731708873681.51bf57bcf1afa9f625176363284d17f7.} 2024-11-15T22:14:42,449 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ccc,1731708873681.2424ff4b6d907aafacef6f661d88ef95. 2024-11-15T22:14:42,449 INFO [RS:0;e611192d6313:38729 {}] regionserver.HRegionServer(3091): Received CLOSE for c1cecce90107a4c37b7d91651975993f 2024-11-15T22:14:42,449 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1731708873681.2424ff4b6d907aafacef6f661d88ef95. 2024-11-15T22:14:42,449 INFO [RS:0;e611192d6313:38729 {}] regionserver.HRegionServer(3091): Received CLOSE for 13c3457b9645c9a8181ed9b4551962ad 2024-11-15T22:14:42,450 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1731708873681.2424ff4b6d907aafacef6f661d88ef95. after waiting 1 ms 2024-11-15T22:14:42,450 INFO [RS:0;e611192d6313:38729 {}] regionserver.HRegionServer(3091): Received CLOSE for 819421a6406c0ebb5635c399b2c0303d 2024-11-15T22:14:42,450 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1731708873681.2424ff4b6d907aafacef6f661d88ef95. 2024-11-15T22:14:42,450 INFO [RS:0;e611192d6313:38729 {}] regionserver.HRegionServer(3091): Received CLOSE for de9977b187fcaabda585b015a568e905 2024-11-15T22:14:42,450 INFO [RS:0;e611192d6313:38729 {}] regionserver.HRegionServer(3091): Received CLOSE for a602f0e8bd42c685584b8b985174ddea 2024-11-15T22:14:42,450 INFO [RS:0;e611192d6313:38729 {}] regionserver.HRegionServer(3091): Received CLOSE for f9e6155d6c82d9d9a5c9449be6d7ce5f 2024-11-15T22:14:42,450 INFO [RS:0;e611192d6313:38729 {}] regionserver.HRegionServer(3091): Received CLOSE for 6efdb15046141ab0cd574540ef2ac2c2 2024-11-15T22:14:42,450 DEBUG [RS:1;e611192d6313:41231 {}] regionserver.HRegionServer(1351): Waiting on 1959cb95bb21ecdc677f7f5290173984, 2a159d388ea2138ee8e2bed061a785ce, 36b8b3bed3adb94f5901f1f3809b64fa, 4381d14830a288bee3aeb42195dd8768, 51bf57bcf1afa9f625176363284d17f7, 83048e0be415b190a19f64bc2e49d3fe, 93deaad3e3229244196e8daa9478371a, a7fb991e3993a690fe7a0a1b064d5f5d, e87a82b283b7251342a664ba3876c98e, f64d8a749084ba25ea1b77db9a0d9f17 2024-11-15T22:14:42,450 INFO [RS:0;e611192d6313:38729 {}] regionserver.HRegionServer(3091): Received CLOSE for ecaeddccb9b2987f43bb050f0a105976 2024-11-15T22:14:42,450 INFO [RS:0;e611192d6313:38729 {}] regionserver.HRegionServer(3091): Received CLOSE for 1732b30f3d981289760d152f788aff6f 2024-11-15T22:14:42,450 INFO [RS:0;e611192d6313:38729 {}] regionserver.HRegionServer(959): stopping server e611192d6313,38729,1731708868111 2024-11-15T22:14:42,450 INFO [RS:0;e611192d6313:38729 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-15T22:14:42,450 INFO [RS:0;e611192d6313:38729 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;e611192d6313:38729. 2024-11-15T22:14:42,450 DEBUG [RS:0;e611192d6313:38729 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T22:14:42,450 DEBUG [RS:0;e611192d6313:38729 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:14:42,450 INFO [RS:0;e611192d6313:38729 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-15T22:14:42,450 INFO [RS:0;e611192d6313:38729 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-15T22:14:42,450 INFO [RS:0;e611192d6313:38729 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-15T22:14:42,450 INFO [RS:0;e611192d6313:38729 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-15T22:14:42,451 INFO [RS:0;e611192d6313:38729 {}] regionserver.HRegionServer(1321): Waiting on 14 regions to close 2024-11-15T22:14:42,451 DEBUG [RS:0;e611192d6313:38729 {}] regionserver.HRegionServer(1351): Waiting on 13c3457b9645c9a8181ed9b4551962ad, 1588230740, 1732b30f3d981289760d152f788aff6f, 2424ff4b6d907aafacef6f661d88ef95, 6efdb15046141ab0cd574540ef2ac2c2, 7502fc847b4b97e1ace9a22ec1c068e0, 819421a6406c0ebb5635c399b2c0303d, a602f0e8bd42c685584b8b985174ddea, c1cecce90107a4c37b7d91651975993f, c56d152c07a95a189ac65341ff4db1fa, de9977b187fcaabda585b015a568e905, ecaeddccb9b2987f43bb050f0a105976, f2783381a48c13c2f75c07e5029903cb, f9e6155d6c82d9d9a5c9449be6d7ce5f 2024-11-15T22:14:42,451 DEBUG [RS_CLOSE_META-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-15T22:14:42,451 INFO [RS_CLOSE_META-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-15T22:14:42,451 DEBUG [RS_CLOSE_META-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-15T22:14:42,451 DEBUG [RS_CLOSE_META-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-15T22:14:42,451 DEBUG [RS_CLOSE_META-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-15T22:14:42,451 INFO [RS_CLOSE_META-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.64 KB heapSize=5.42 KB 2024-11-15T22:14:42,456 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/2424ff4b6d907aafacef6f661d88ef95/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T22:14:42,457 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ccc,1731708873681.2424ff4b6d907aafacef6f661d88ef95. 2024-11-15T22:14:42,457 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2424ff4b6d907aafacef6f661d88ef95: Waiting for close lock at 1731708882449Running coprocessor pre-close hooks at 1731708882449Disabling compacts and flushes for region at 1731708882449Disabling writes for close at 1731708882450 (+1 ms)Writing region close event to WAL at 1731708882451 (+1 ms)Running coprocessor post-close hooks at 1731708882457 (+6 ms)Closed at 1731708882457 2024-11-15T22:14:42,457 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/36b8b3bed3adb94f5901f1f3809b64fa/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T22:14:42,457 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ccc,1731708873681.2424ff4b6d907aafacef6f661d88ef95. 2024-11-15T22:14:42,457 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c56d152c07a95a189ac65341ff4db1fa, disabling compactions & flushes 2024-11-15T22:14:42,457 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,nnn,1731708873681.c56d152c07a95a189ac65341ff4db1fa. 2024-11-15T22:14:42,457 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1731708873681.c56d152c07a95a189ac65341ff4db1fa. 2024-11-15T22:14:42,457 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1731708873681.c56d152c07a95a189ac65341ff4db1fa. after waiting 0 ms 2024-11-15T22:14:42,457 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1731708873681.c56d152c07a95a189ac65341ff4db1fa. 2024-11-15T22:14:42,458 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,jjj,1731708873681.36b8b3bed3adb94f5901f1f3809b64fa. 2024-11-15T22:14:42,458 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 36b8b3bed3adb94f5901f1f3809b64fa: Waiting for close lock at 1731708882448Running coprocessor pre-close hooks at 1731708882449 (+1 ms)Disabling compacts and flushes for region at 1731708882449Disabling writes for close at 1731708882449Writing region close event to WAL at 1731708882451 (+2 ms)Running coprocessor post-close hooks at 1731708882458 (+7 ms)Closed at 1731708882458 2024-11-15T22:14:42,458 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,jjj,1731708873681.36b8b3bed3adb94f5901f1f3809b64fa. 2024-11-15T22:14:42,458 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a7fb991e3993a690fe7a0a1b064d5f5d, disabling compactions & flushes 2024-11-15T22:14:42,458 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,xxx,1731708873681.a7fb991e3993a690fe7a0a1b064d5f5d. 2024-11-15T22:14:42,459 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1731708873681.a7fb991e3993a690fe7a0a1b064d5f5d. 2024-11-15T22:14:42,459 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1731708873681.a7fb991e3993a690fe7a0a1b064d5f5d. after waiting 0 ms 2024-11-15T22:14:42,459 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1731708873681.a7fb991e3993a690fe7a0a1b064d5f5d. 2024-11-15T22:14:42,461 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/c56d152c07a95a189ac65341ff4db1fa/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T22:14:42,462 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,nnn,1731708873681.c56d152c07a95a189ac65341ff4db1fa. 2024-11-15T22:14:42,462 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c56d152c07a95a189ac65341ff4db1fa: Waiting for close lock at 1731708882457Running coprocessor pre-close hooks at 1731708882457Disabling compacts and flushes for region at 1731708882457Disabling writes for close at 1731708882457Writing region close event to WAL at 1731708882458 (+1 ms)Running coprocessor post-close hooks at 1731708882462 (+4 ms)Closed at 1731708882462 2024-11-15T22:14:42,462 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/a7fb991e3993a690fe7a0a1b064d5f5d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T22:14:42,462 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,nnn,1731708873681.c56d152c07a95a189ac65341ff4db1fa. 2024-11-15T22:14:42,462 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f2783381a48c13c2f75c07e5029903cb, disabling compactions & flushes 2024-11-15T22:14:42,462 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,,1731708873681.f2783381a48c13c2f75c07e5029903cb. 2024-11-15T22:14:42,462 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1731708873681.f2783381a48c13c2f75c07e5029903cb. 2024-11-15T22:14:42,462 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,,1731708873681.f2783381a48c13c2f75c07e5029903cb. after waiting 0 ms 2024-11-15T22:14:42,462 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,,1731708873681.f2783381a48c13c2f75c07e5029903cb. 2024-11-15T22:14:42,463 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,xxx,1731708873681.a7fb991e3993a690fe7a0a1b064d5f5d. 2024-11-15T22:14:42,463 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a7fb991e3993a690fe7a0a1b064d5f5d: Waiting for close lock at 1731708882458Running coprocessor pre-close hooks at 1731708882458Disabling compacts and flushes for region at 1731708882458Disabling writes for close at 1731708882459 (+1 ms)Writing region close event to WAL at 1731708882459Running coprocessor post-close hooks at 1731708882463 (+4 ms)Closed at 1731708882463 2024-11-15T22:14:42,463 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,xxx,1731708873681.a7fb991e3993a690fe7a0a1b064d5f5d. 2024-11-15T22:14:42,463 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f64d8a749084ba25ea1b77db9a0d9f17, disabling compactions & flushes 2024-11-15T22:14:42,463 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,mmm,1731708873681.f64d8a749084ba25ea1b77db9a0d9f17. 2024-11-15T22:14:42,463 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1731708873681.f64d8a749084ba25ea1b77db9a0d9f17. 2024-11-15T22:14:42,463 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1731708873681.f64d8a749084ba25ea1b77db9a0d9f17. after waiting 0 ms 2024-11-15T22:14:42,463 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1731708873681.f64d8a749084ba25ea1b77db9a0d9f17. 2024-11-15T22:14:42,463 DEBUG [RS_CLOSE_META-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/hbase/meta/1588230740/.tmp/info/5b487ca5799142a28b63fc9e9867e7e8 is 138, key is test,bbb,1731708873681.897a7bb57a002d365489ce583eb7f84f./info:regioninfo/1731708881297/Put/seqid=0 2024-11-15T22:14:42,466 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/f2783381a48c13c2f75c07e5029903cb/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T22:14:42,466 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/f64d8a749084ba25ea1b77db9a0d9f17/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T22:14:42,467 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,mmm,1731708873681.f64d8a749084ba25ea1b77db9a0d9f17. 2024-11-15T22:14:42,467 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,,1731708873681.f2783381a48c13c2f75c07e5029903cb. 2024-11-15T22:14:42,467 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f64d8a749084ba25ea1b77db9a0d9f17: Waiting for close lock at 1731708882463Running coprocessor pre-close hooks at 1731708882463Disabling compacts and flushes for region at 1731708882463Disabling writes for close at 1731708882463Writing region close event to WAL at 1731708882464 (+1 ms)Running coprocessor post-close hooks at 1731708882467 (+3 ms)Closed at 1731708882467 2024-11-15T22:14:42,467 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f2783381a48c13c2f75c07e5029903cb: Waiting for close lock at 1731708882462Running coprocessor pre-close hooks at 1731708882462Disabling compacts and flushes for region at 1731708882462Disabling writes for close at 1731708882462Writing region close event to WAL at 1731708882463 (+1 ms)Running coprocessor post-close hooks at 1731708882467 (+4 ms)Closed at 1731708882467 2024-11-15T22:14:42,467 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,mmm,1731708873681.f64d8a749084ba25ea1b77db9a0d9f17. 2024-11-15T22:14:42,467 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,,1731708873681.f2783381a48c13c2f75c07e5029903cb. 2024-11-15T22:14:42,467 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1959cb95bb21ecdc677f7f5290173984, disabling compactions & flushes 2024-11-15T22:14:42,467 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7502fc847b4b97e1ace9a22ec1c068e0, disabling compactions & flushes 2024-11-15T22:14:42,467 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ggg,1731708873681.1959cb95bb21ecdc677f7f5290173984. 2024-11-15T22:14:42,467 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,rrr,1731708873681.7502fc847b4b97e1ace9a22ec1c068e0. 2024-11-15T22:14:42,467 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1731708873681.7502fc847b4b97e1ace9a22ec1c068e0. 2024-11-15T22:14:42,467 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1731708873681.1959cb95bb21ecdc677f7f5290173984. 2024-11-15T22:14:42,467 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1731708873681.7502fc847b4b97e1ace9a22ec1c068e0. after waiting 0 ms 2024-11-15T22:14:42,467 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1731708873681.1959cb95bb21ecdc677f7f5290173984. after waiting 0 ms 2024-11-15T22:14:42,467 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1731708873681.7502fc847b4b97e1ace9a22ec1c068e0. 2024-11-15T22:14:42,467 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1731708873681.1959cb95bb21ecdc677f7f5290173984. 2024-11-15T22:14:42,473 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/1959cb95bb21ecdc677f7f5290173984/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T22:14:42,473 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/7502fc847b4b97e1ace9a22ec1c068e0/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T22:14:42,474 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ggg,1731708873681.1959cb95bb21ecdc677f7f5290173984. 2024-11-15T22:14:42,474 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,rrr,1731708873681.7502fc847b4b97e1ace9a22ec1c068e0. 2024-11-15T22:14:42,474 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7502fc847b4b97e1ace9a22ec1c068e0: Waiting for close lock at 1731708882467Running coprocessor pre-close hooks at 1731708882467Disabling compacts and flushes for region at 1731708882467Disabling writes for close at 1731708882467Writing region close event to WAL at 1731708882468 (+1 ms)Running coprocessor post-close hooks at 1731708882474 (+6 ms)Closed at 1731708882474 2024-11-15T22:14:42,474 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1959cb95bb21ecdc677f7f5290173984: Waiting for close lock at 1731708882467Running coprocessor pre-close hooks at 1731708882467Disabling compacts and flushes for region at 1731708882467Disabling writes for close at 1731708882467Writing region close event to WAL at 1731708882468 (+1 ms)Running coprocessor post-close hooks at 1731708882474 (+6 ms)Closed at 1731708882474 2024-11-15T22:14:42,474 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,rrr,1731708873681.7502fc847b4b97e1ace9a22ec1c068e0. 2024-11-15T22:14:42,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33631 is added to blk_1073741869_1045 (size=8177) 2024-11-15T22:14:42,474 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ggg,1731708873681.1959cb95bb21ecdc677f7f5290173984. 2024-11-15T22:14:42,475 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 83048e0be415b190a19f64bc2e49d3fe, disabling compactions & flushes 2024-11-15T22:14:42,475 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c1cecce90107a4c37b7d91651975993f, disabling compactions & flushes 2024-11-15T22:14:42,475 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ppp,1731708873681.83048e0be415b190a19f64bc2e49d3fe. 2024-11-15T22:14:42,475 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,yyy,1731708873681.c1cecce90107a4c37b7d91651975993f. 2024-11-15T22:14:42,475 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1731708873681.83048e0be415b190a19f64bc2e49d3fe. 2024-11-15T22:14:42,475 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1731708873681.c1cecce90107a4c37b7d91651975993f. 2024-11-15T22:14:42,475 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1731708873681.83048e0be415b190a19f64bc2e49d3fe. after waiting 0 ms 2024-11-15T22:14:42,475 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1731708873681.c1cecce90107a4c37b7d91651975993f. after waiting 0 ms 2024-11-15T22:14:42,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741869_1045 (size=8177) 2024-11-15T22:14:42,475 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1731708873681.83048e0be415b190a19f64bc2e49d3fe. 2024-11-15T22:14:42,475 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1731708873681.c1cecce90107a4c37b7d91651975993f. 2024-11-15T22:14:42,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39269 is added to blk_1073741869_1045 (size=8177) 2024-11-15T22:14:42,476 INFO [RS_CLOSE_META-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.64 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/hbase/meta/1588230740/.tmp/info/5b487ca5799142a28b63fc9e9867e7e8 2024-11-15T22:14:42,479 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/c1cecce90107a4c37b7d91651975993f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T22:14:42,479 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/83048e0be415b190a19f64bc2e49d3fe/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T22:14:42,480 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,yyy,1731708873681.c1cecce90107a4c37b7d91651975993f. 2024-11-15T22:14:42,480 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c1cecce90107a4c37b7d91651975993f: Waiting for close lock at 1731708882475Running coprocessor pre-close hooks at 1731708882475Disabling compacts and flushes for region at 1731708882475Disabling writes for close at 1731708882475Writing region close event to WAL at 1731708882476 (+1 ms)Running coprocessor post-close hooks at 1731708882480 (+4 ms)Closed at 1731708882480 2024-11-15T22:14:42,480 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,yyy,1731708873681.c1cecce90107a4c37b7d91651975993f. 2024-11-15T22:14:42,480 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 13c3457b9645c9a8181ed9b4551962ad, disabling compactions & flushes 2024-11-15T22:14:42,480 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,hhh,1731708873681.13c3457b9645c9a8181ed9b4551962ad. 2024-11-15T22:14:42,480 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ppp,1731708873681.83048e0be415b190a19f64bc2e49d3fe. 2024-11-15T22:14:42,480 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1731708873681.13c3457b9645c9a8181ed9b4551962ad. 2024-11-15T22:14:42,480 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 83048e0be415b190a19f64bc2e49d3fe: Waiting for close lock at 1731708882475Running coprocessor pre-close hooks at 1731708882475Disabling compacts and flushes for region at 1731708882475Disabling writes for close at 1731708882475Writing region close event to WAL at 1731708882475Running coprocessor post-close hooks at 1731708882480 (+5 ms)Closed at 1731708882480 2024-11-15T22:14:42,480 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1731708873681.13c3457b9645c9a8181ed9b4551962ad. after waiting 0 ms 2024-11-15T22:14:42,480 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1731708873681.13c3457b9645c9a8181ed9b4551962ad. 2024-11-15T22:14:42,480 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ppp,1731708873681.83048e0be415b190a19f64bc2e49d3fe. 2024-11-15T22:14:42,480 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e87a82b283b7251342a664ba3876c98e, disabling compactions & flushes 2024-11-15T22:14:42,480 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,uuu,1731708873681.e87a82b283b7251342a664ba3876c98e. 2024-11-15T22:14:42,480 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1731708873681.e87a82b283b7251342a664ba3876c98e. 2024-11-15T22:14:42,480 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1731708873681.e87a82b283b7251342a664ba3876c98e. after waiting 0 ms 2024-11-15T22:14:42,481 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1731708873681.e87a82b283b7251342a664ba3876c98e. 2024-11-15T22:14:42,484 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/13c3457b9645c9a8181ed9b4551962ad/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T22:14:42,484 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/e87a82b283b7251342a664ba3876c98e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T22:14:42,485 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,hhh,1731708873681.13c3457b9645c9a8181ed9b4551962ad. 2024-11-15T22:14:42,485 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 13c3457b9645c9a8181ed9b4551962ad: Waiting for close lock at 1731708882480Running coprocessor pre-close hooks at 1731708882480Disabling compacts and flushes for region at 1731708882480Disabling writes for close at 1731708882480Writing region close event to WAL at 1731708882481 (+1 ms)Running coprocessor post-close hooks at 1731708882485 (+4 ms)Closed at 1731708882485 2024-11-15T22:14:42,485 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,uuu,1731708873681.e87a82b283b7251342a664ba3876c98e. 2024-11-15T22:14:42,485 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e87a82b283b7251342a664ba3876c98e: Waiting for close lock at 1731708882480Running coprocessor pre-close hooks at 1731708882480Disabling compacts and flushes for region at 1731708882480Disabling writes for close at 1731708882480Writing region close event to WAL at 1731708882481 (+1 ms)Running coprocessor post-close hooks at 1731708882485 (+4 ms)Closed at 1731708882485 2024-11-15T22:14:42,485 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,hhh,1731708873681.13c3457b9645c9a8181ed9b4551962ad. 2024-11-15T22:14:42,485 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,uuu,1731708873681.e87a82b283b7251342a664ba3876c98e. 2024-11-15T22:14:42,485 DEBUG [RS_CLOSE_META-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/hbase/meta/1588230740/.tmp/info/5b487ca5799142a28b63fc9e9867e7e8 as hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/hbase/meta/1588230740/info/5b487ca5799142a28b63fc9e9867e7e8 2024-11-15T22:14:42,485 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4381d14830a288bee3aeb42195dd8768, disabling compactions & flushes 2024-11-15T22:14:42,485 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 819421a6406c0ebb5635c399b2c0303d, disabling compactions & flushes 2024-11-15T22:14:42,485 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ttt,1731708873681.4381d14830a288bee3aeb42195dd8768. 2024-11-15T22:14:42,485 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,sss,1731708873681.819421a6406c0ebb5635c399b2c0303d. 2024-11-15T22:14:42,485 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1731708873681.4381d14830a288bee3aeb42195dd8768. 2024-11-15T22:14:42,486 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1731708873681.819421a6406c0ebb5635c399b2c0303d. 2024-11-15T22:14:42,486 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1731708873681.4381d14830a288bee3aeb42195dd8768. after waiting 0 ms 2024-11-15T22:14:42,486 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,sss,1731708873681.819421a6406c0ebb5635c399b2c0303d. after waiting 0 ms 2024-11-15T22:14:42,486 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1731708873681.4381d14830a288bee3aeb42195dd8768. 2024-11-15T22:14:42,486 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,sss,1731708873681.819421a6406c0ebb5635c399b2c0303d. 2024-11-15T22:14:42,488 INFO [regionserver/e611192d6313:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T22:14:42,490 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/4381d14830a288bee3aeb42195dd8768/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T22:14:42,490 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/819421a6406c0ebb5635c399b2c0303d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T22:14:42,490 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,sss,1731708873681.819421a6406c0ebb5635c399b2c0303d. 2024-11-15T22:14:42,491 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 819421a6406c0ebb5635c399b2c0303d: Waiting for close lock at 1731708882485Running coprocessor pre-close hooks at 1731708882485Disabling compacts and flushes for region at 1731708882485Disabling writes for close at 1731708882486 (+1 ms)Writing region close event to WAL at 1731708882486Running coprocessor post-close hooks at 1731708882490 (+4 ms)Closed at 1731708882490 2024-11-15T22:14:42,491 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,sss,1731708873681.819421a6406c0ebb5635c399b2c0303d. 2024-11-15T22:14:42,491 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ttt,1731708873681.4381d14830a288bee3aeb42195dd8768. 2024-11-15T22:14:42,491 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4381d14830a288bee3aeb42195dd8768: Waiting for close lock at 1731708882485Running coprocessor pre-close hooks at 1731708882485Disabling compacts and flushes for region at 1731708882485Disabling writes for close at 1731708882486 (+1 ms)Writing region close event to WAL at 1731708882486Running coprocessor post-close hooks at 1731708882491 (+5 ms)Closed at 1731708882491 2024-11-15T22:14:42,491 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ttt,1731708873681.4381d14830a288bee3aeb42195dd8768. 2024-11-15T22:14:42,491 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing de9977b187fcaabda585b015a568e905, disabling compactions & flushes 2024-11-15T22:14:42,491 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,lll,1731708873681.de9977b187fcaabda585b015a568e905. 2024-11-15T22:14:42,491 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 93deaad3e3229244196e8daa9478371a, disabling compactions & flushes 2024-11-15T22:14:42,491 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1731708873681.de9977b187fcaabda585b015a568e905. 2024-11-15T22:14:42,491 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,eee,1731708873681.93deaad3e3229244196e8daa9478371a. 2024-11-15T22:14:42,491 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1731708873681.93deaad3e3229244196e8daa9478371a. 2024-11-15T22:14:42,491 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,lll,1731708873681.de9977b187fcaabda585b015a568e905. after waiting 0 ms 2024-11-15T22:14:42,491 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,eee,1731708873681.93deaad3e3229244196e8daa9478371a. after waiting 0 ms 2024-11-15T22:14:42,491 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,eee,1731708873681.93deaad3e3229244196e8daa9478371a. 2024-11-15T22:14:42,491 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,lll,1731708873681.de9977b187fcaabda585b015a568e905. 2024-11-15T22:14:42,494 INFO [RS_CLOSE_META-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/hbase/meta/1588230740/info/5b487ca5799142a28b63fc9e9867e7e8, entries=24, sequenceid=75, filesize=8.0 K 2024-11-15T22:14:42,495 INFO [RS_CLOSE_META-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.64 KB/2708, heapSize ~4.66 KB/4768, currentSize=0 B/0 for 1588230740 in 44ms, sequenceid=75, compaction requested=false 2024-11-15T22:14:42,495 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/93deaad3e3229244196e8daa9478371a/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-15T22:14:42,496 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/de9977b187fcaabda585b015a568e905/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T22:14:42,496 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,eee,1731708873681.93deaad3e3229244196e8daa9478371a. 2024-11-15T22:14:42,496 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,lll,1731708873681.de9977b187fcaabda585b015a568e905. 2024-11-15T22:14:42,496 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for de9977b187fcaabda585b015a568e905: Waiting for close lock at 1731708882491Running coprocessor pre-close hooks at 1731708882491Disabling compacts and flushes for region at 1731708882491Disabling writes for close at 1731708882491Writing region close event to WAL at 1731708882492 (+1 ms)Running coprocessor post-close hooks at 1731708882496 (+4 ms)Closed at 1731708882496 2024-11-15T22:14:42,496 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 93deaad3e3229244196e8daa9478371a: Waiting for close lock at 1731708882491Running coprocessor pre-close hooks at 1731708882491Disabling compacts and flushes for region at 1731708882491Disabling writes for close at 1731708882491Writing region close event to WAL at 1731708882492 (+1 ms)Running coprocessor post-close hooks at 1731708882496 (+4 ms)Closed at 1731708882496 2024-11-15T22:14:42,496 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,eee,1731708873681.93deaad3e3229244196e8daa9478371a. 2024-11-15T22:14:42,496 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,lll,1731708873681.de9977b187fcaabda585b015a568e905. 2024-11-15T22:14:42,497 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 51bf57bcf1afa9f625176363284d17f7, disabling compactions & flushes 2024-11-15T22:14:42,497 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a602f0e8bd42c685584b8b985174ddea, disabling compactions & flushes 2024-11-15T22:14:42,497 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ddd,1731708873681.51bf57bcf1afa9f625176363284d17f7. 2024-11-15T22:14:42,497 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ooo,1731708873681.a602f0e8bd42c685584b8b985174ddea. 2024-11-15T22:14:42,497 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1731708873681.51bf57bcf1afa9f625176363284d17f7. 2024-11-15T22:14:42,497 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1731708873681.a602f0e8bd42c685584b8b985174ddea. 2024-11-15T22:14:42,497 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1731708873681.51bf57bcf1afa9f625176363284d17f7. after waiting 0 ms 2024-11-15T22:14:42,497 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1731708873681.51bf57bcf1afa9f625176363284d17f7. 2024-11-15T22:14:42,497 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1731708873681.a602f0e8bd42c685584b8b985174ddea. after waiting 0 ms 2024-11-15T22:14:42,497 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1731708873681.a602f0e8bd42c685584b8b985174ddea. 2024-11-15T22:14:42,500 DEBUG [RS_CLOSE_META-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/hbase/meta/1588230740/recovered.edits/78.seqid, newMaxSeqId=78, maxSeqId=64 2024-11-15T22:14:42,501 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/a602f0e8bd42c685584b8b985174ddea/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T22:14:42,501 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/51bf57bcf1afa9f625176363284d17f7/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T22:14:42,501 DEBUG [RS_CLOSE_META-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-15T22:14:42,501 INFO [RS_CLOSE_META-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-15T22:14:42,501 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ooo,1731708873681.a602f0e8bd42c685584b8b985174ddea. 2024-11-15T22:14:42,501 DEBUG [RS_CLOSE_META-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731708882451Running coprocessor pre-close hooks at 1731708882451Disabling compacts and flushes for region at 1731708882451Disabling writes for close at 1731708882451Obtaining lock to block concurrent updates at 1731708882451Preparing flush snapshotting stores in 1588230740 at 1731708882451Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=2708, getHeapSize=5488, getOffHeapSize=0, getCellsCount=24 at 1731708882456 (+5 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731708882457 (+1 ms)Flushing 1588230740/info: creating writer at 1731708882457Flushing 1588230740/info: appending metadata at 1731708882462 (+5 ms)Flushing 1588230740/info: closing flushed file at 1731708882462Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@38b6a9d6: reopening flushed file at 1731708882484 (+22 ms)Finished flush of dataSize ~2.64 KB/2708, heapSize ~4.66 KB/4768, currentSize=0 B/0 for 1588230740 in 44ms, sequenceid=75, compaction requested=false at 1731708882495 (+11 ms)Writing region close event to WAL at 1731708882497 (+2 ms)Running coprocessor post-close hooks at 1731708882501 (+4 ms)Closed at 1731708882501 2024-11-15T22:14:42,501 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a602f0e8bd42c685584b8b985174ddea: Waiting for close lock at 1731708882497Running coprocessor pre-close hooks at 1731708882497Disabling compacts and flushes for region at 1731708882497Disabling writes for close at 1731708882497Writing region close event to WAL at 1731708882497Running coprocessor post-close hooks at 1731708882501 (+4 ms)Closed at 1731708882501 2024-11-15T22:14:42,501 DEBUG [RS_CLOSE_META-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-15T22:14:42,501 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ddd,1731708873681.51bf57bcf1afa9f625176363284d17f7. 2024-11-15T22:14:42,501 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ooo,1731708873681.a602f0e8bd42c685584b8b985174ddea. 2024-11-15T22:14:42,502 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 51bf57bcf1afa9f625176363284d17f7: Waiting for close lock at 1731708882497Running coprocessor pre-close hooks at 1731708882497Disabling compacts and flushes for region at 1731708882497Disabling writes for close at 1731708882497Writing region close event to WAL at 1731708882497Running coprocessor post-close hooks at 1731708882501 (+4 ms)Closed at 1731708882501 2024-11-15T22:14:42,502 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f9e6155d6c82d9d9a5c9449be6d7ce5f, disabling compactions & flushes 2024-11-15T22:14:42,502 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ddd,1731708873681.51bf57bcf1afa9f625176363284d17f7. 2024-11-15T22:14:42,502 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,vvv,1731708873681.f9e6155d6c82d9d9a5c9449be6d7ce5f. 2024-11-15T22:14:42,502 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1731708873681.f9e6155d6c82d9d9a5c9449be6d7ce5f. 2024-11-15T22:14:42,502 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1731708873681.f9e6155d6c82d9d9a5c9449be6d7ce5f. after waiting 0 ms 2024-11-15T22:14:42,502 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1731708873681.f9e6155d6c82d9d9a5c9449be6d7ce5f. 2024-11-15T22:14:42,506 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/f9e6155d6c82d9d9a5c9449be6d7ce5f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T22:14:42,506 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,vvv,1731708873681.f9e6155d6c82d9d9a5c9449be6d7ce5f. 2024-11-15T22:14:42,506 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f9e6155d6c82d9d9a5c9449be6d7ce5f: Waiting for close lock at 1731708882502Running coprocessor pre-close hooks at 1731708882502Disabling compacts and flushes for region at 1731708882502Disabling writes for close at 1731708882502Writing region close event to WAL at 1731708882502Running coprocessor post-close hooks at 1731708882506 (+4 ms)Closed at 1731708882506 2024-11-15T22:14:42,506 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,vvv,1731708873681.f9e6155d6c82d9d9a5c9449be6d7ce5f. 2024-11-15T22:14:42,507 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6efdb15046141ab0cd574540ef2ac2c2, disabling compactions & flushes 2024-11-15T22:14:42,507 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,zzz,1731708873681.6efdb15046141ab0cd574540ef2ac2c2. 2024-11-15T22:14:42,507 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1731708873681.6efdb15046141ab0cd574540ef2ac2c2. 2024-11-15T22:14:42,507 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1731708873681.6efdb15046141ab0cd574540ef2ac2c2. after waiting 0 ms 2024-11-15T22:14:42,507 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1731708873681.6efdb15046141ab0cd574540ef2ac2c2. 2024-11-15T22:14:42,511 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/6efdb15046141ab0cd574540ef2ac2c2/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T22:14:42,511 INFO [regionserver/e611192d6313:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T22:14:42,511 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,zzz,1731708873681.6efdb15046141ab0cd574540ef2ac2c2. 2024-11-15T22:14:42,511 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6efdb15046141ab0cd574540ef2ac2c2: Waiting for close lock at 1731708882507Running coprocessor pre-close hooks at 1731708882507Disabling compacts and flushes for region at 1731708882507Disabling writes for close at 1731708882507Writing region close event to WAL at 1731708882507Running coprocessor post-close hooks at 1731708882511 (+4 ms)Closed at 1731708882511 2024-11-15T22:14:42,512 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,zzz,1731708873681.6efdb15046141ab0cd574540ef2ac2c2. 2024-11-15T22:14:42,512 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ecaeddccb9b2987f43bb050f0a105976, disabling compactions & flushes 2024-11-15T22:14:42,512 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,fff,1731708873681.ecaeddccb9b2987f43bb050f0a105976. 2024-11-15T22:14:42,512 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1731708873681.ecaeddccb9b2987f43bb050f0a105976. 2024-11-15T22:14:42,512 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,fff,1731708873681.ecaeddccb9b2987f43bb050f0a105976. after waiting 0 ms 2024-11-15T22:14:42,512 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,fff,1731708873681.ecaeddccb9b2987f43bb050f0a105976. 2024-11-15T22:14:42,516 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/ecaeddccb9b2987f43bb050f0a105976/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T22:14:42,517 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,fff,1731708873681.ecaeddccb9b2987f43bb050f0a105976. 2024-11-15T22:14:42,517 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ecaeddccb9b2987f43bb050f0a105976: Waiting for close lock at 1731708882512Running coprocessor pre-close hooks at 1731708882512Disabling compacts and flushes for region at 1731708882512Disabling writes for close at 1731708882512Writing region close event to WAL at 1731708882512Running coprocessor post-close hooks at 1731708882517 (+5 ms)Closed at 1731708882517 2024-11-15T22:14:42,517 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,fff,1731708873681.ecaeddccb9b2987f43bb050f0a105976. 2024-11-15T22:14:42,517 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1732b30f3d981289760d152f788aff6f, disabling compactions & flushes 2024-11-15T22:14:42,517 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,iii,1731708873681.1732b30f3d981289760d152f788aff6f. 2024-11-15T22:14:42,517 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1731708873681.1732b30f3d981289760d152f788aff6f. 2024-11-15T22:14:42,518 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,iii,1731708873681.1732b30f3d981289760d152f788aff6f. after waiting 1 ms 2024-11-15T22:14:42,518 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,iii,1731708873681.1732b30f3d981289760d152f788aff6f. 2024-11-15T22:14:42,522 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/1732b30f3d981289760d152f788aff6f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T22:14:42,523 INFO [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,iii,1731708873681.1732b30f3d981289760d152f788aff6f. 2024-11-15T22:14:42,523 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1732b30f3d981289760d152f788aff6f: Waiting for close lock at 1731708882517Running coprocessor pre-close hooks at 1731708882517Disabling compacts and flushes for region at 1731708882517Disabling writes for close at 1731708882518 (+1 ms)Writing region close event to WAL at 1731708882518Running coprocessor post-close hooks at 1731708882523 (+5 ms)Closed at 1731708882523 2024-11-15T22:14:42,523 DEBUG [RS_CLOSE_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,iii,1731708873681.1732b30f3d981289760d152f788aff6f. 2024-11-15T22:14:42,561 INFO [regionserver/e611192d6313:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-15T22:14:42,562 INFO [regionserver/e611192d6313:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-15T22:14:42,563 INFO [regionserver/e611192d6313:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-15T22:14:42,563 INFO [regionserver/e611192d6313:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-15T22:14:42,600 WARN [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,kkk,1731708873681.2a159d388ea2138ee8e2bed061a785ce., aborting... java.io.IOException: Failed to report opened region to master: test,kkk,1731708873681.2a159d388ea2138ee8e2bed061a785ce. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:42,601 ERROR [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2440): ***** ABORTING region server e611192d6313,41231,1731708869041: Failed to open region test,kkk,1731708873681.2a159d388ea2138ee8e2bed061a785ce. and can not recover ***** java.io.IOException: Failed to report opened region to master: test,kkk,1731708873681.2a159d388ea2138ee8e2bed061a785ce. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:42,601 ERROR [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-15T22:14:42,601 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-15T22:14:42,601 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-15T22:14:42,602 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-15T22:14:42,603 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-15T22:14:42,607 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 520962024 }, "NonHeapMemoryUsage": { "committed": 162660352, "init": 7667712, "max": -1, "used": 160132968 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "e611192d6313", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 2, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 34393, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 100, "ProcessCallTime_min": 0, "ProcessCallTime_max": 28, "ProcessCallTime_mean": 4, "ProcessCallTime_25th_percentile": 7, "ProcessCallTime_median": 14, "ProcessCallTime_75th_percentile": 21, "ProcessCallTime_90th_percentile": 25, "ProcessCallTime_95th_percentile": 26, "ProcessCallTime_98th_percentile": 27, "ProcessCallTime_99th_percentile": 27, "ProcessCallTime_99.9th_percentile": 27, "ProcessCallTime_TimeRangeCount_0-1": 100, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 100, "QueueCallTime_min": 0, "QueueCallTime_max": 67, "QueueCallTime_mean": 7, "QueueCallTime_25th_percentile": 16, "QueueCallTime_median": 33, "QueueCallTime_75th_percentile": 50, "QueueCallTime_90th_percentile": 60, "QueueCallTime_95th_percentile": 63, "QueueCallTime_98th_percentile": 65, "QueueCallTime_99th_percentile": 66, "QueueCallTime_99.9th_percentile": 66, "QueueCallTime_TimeRangeCount_0-1": 100, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 100, "TotalCallTime_min": 0, "TotalCallTime_max": 72, "TotalCallTime_mean": 12, "TotalCallTime_25th_percentile": 18, "TotalCallTime_median": 36, "TotalCallTime_75th_percentile": 54, "TotalCallTime_90th_percentile": 64, "TotalCallTime_95th_percentile": 68, "TotalCallTime_98th_percentile": 70, "TotalCallTime_99th_percentile": 71, "TotalCallTime_99.9th_percentile": 71, "TotalCallTime_TimeRangeCount_0-1": 100, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 100, "ResponseSize_min": 0, "ResponseSize_max": 22359, "ResponseSize_mean": 692, "ResponseSize_25th_percentile": 5589, "ResponseSize_median": 11179, "ResponseSize_75th_percentile": 16769, "ResponseSize_90th_percentile": 20123, "ResponseSize_95th_percentile": 21241, "ResponseSize_98th_percentile": 21911, "ResponseSize_99th_percentile": 22135, "ResponseSize_99.9th_percentile": 22336, "ResponseSize_SizeRangeCount_0-10": 100, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 100, "RequestSize_min": 81, "RequestSize_max": 8095, "RequestSize_mean": 311, "RequestSize_25th_percentile": 2084, "RequestSize_median": 4088, "RequestSize_75th_percentile": 6091, "RequestSize_90th_percentile": 7293, "RequestSize_95th_percentile": 7694, "RequestSize_98th_percentile": 7934, "RequestSize_99th_percentile": 8014, "RequestSize_99.9th_percentile": 8086, "RequestSize_SizeRangeCount_0-10": 100, "sentBytes": 55432 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "e611192d6313", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:60621", "tag.serverName": "e611192d6313,41231,1731708869041", "tag.clusterId": "1b4a912b-fe9b-4940-8b4d-31902347eb12", "tag.Context": "regionserver", "tag.Hostname": "e611192d6313", "regionCount": 8, "storeCount": 8, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 2048, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1731708869041, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 0, "localBytesRead": 0, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 0, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 8, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 8, "rpcScanRequestCount": 8, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 8, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "ScanTime_TimeRangeCount_0-1": 8, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 8, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "ScanSize_SizeRangeCount_0-10": 8, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-15T22:14:42,609 WARN [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2470): Unable to report fatal error to master org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=e611192d6313:39087 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:344) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportRSFatalError(RegionServerStatusProtos.java:17290) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.abort(HRegionServer.java:2467) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abortRegionServer(SingleProcessHBaseCluster.java:192) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$2.run(SingleProcessHBaseCluster.java:185) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abort(SingleProcessHBaseCluster.java:182) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.handleException(AssignRegionHandler.java:173) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:106) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=e611192d6313:39087 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 15 more Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 15 more 2024-11-15T22:14:42,609 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(132): Open test,qqq,1731708873681.06b9df8f55d791077e9f7ec55ef6a971. 2024-11-15T22:14:42,609 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7752): Opening region: {ENCODED => 06b9df8f55d791077e9f7ec55ef6a971, NAME => 'test,qqq,1731708873681.06b9df8f55d791077e9f7ec55ef6a971.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-11-15T22:14:42,610 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 06b9df8f55d791077e9f7ec55ef6a971 2024-11-15T22:14:42,610 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(898): Instantiated test,qqq,1731708873681.06b9df8f55d791077e9f7ec55ef6a971.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:42,610 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7794): checking encryption for 06b9df8f55d791077e9f7ec55ef6a971 2024-11-15T22:14:42,610 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7797): checking classloading for 06b9df8f55d791077e9f7ec55ef6a971 2024-11-15T22:14:42,611 INFO [StoreOpener-06b9df8f55d791077e9f7ec55ef6a971-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 06b9df8f55d791077e9f7ec55ef6a971 2024-11-15T22:14:42,612 INFO [StoreOpener-06b9df8f55d791077e9f7ec55ef6a971-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06b9df8f55d791077e9f7ec55ef6a971 columnFamilyName family 2024-11-15T22:14:42,612 DEBUG [StoreOpener-06b9df8f55d791077e9f7ec55ef6a971-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:42,613 INFO [StoreOpener-06b9df8f55d791077e9f7ec55ef6a971-1 {}] regionserver.HStore(327): Store=06b9df8f55d791077e9f7ec55ef6a971/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:14:42,613 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1038): replaying wal for 06b9df8f55d791077e9f7ec55ef6a971 2024-11-15T22:14:42,614 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/06b9df8f55d791077e9f7ec55ef6a971 2024-11-15T22:14:42,616 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/06b9df8f55d791077e9f7ec55ef6a971 2024-11-15T22:14:42,616 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1048): stopping wal replay for 06b9df8f55d791077e9f7ec55ef6a971 2024-11-15T22:14:42,616 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1060): Cleaning up temporary data for 06b9df8f55d791077e9f7ec55ef6a971 2024-11-15T22:14:42,618 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1093): writing seq id for 06b9df8f55d791077e9f7ec55ef6a971 2024-11-15T22:14:42,619 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1114): Opened 06b9df8f55d791077e9f7ec55ef6a971; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73929574, jitterRate=0.10163649916648865}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T22:14:42,619 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 06b9df8f55d791077e9f7ec55ef6a971 2024-11-15T22:14:42,619 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1006): Region open journal for 06b9df8f55d791077e9f7ec55ef6a971: Running coprocessor pre-open hook at 1731708882610Writing region info on filesystem at 1731708882610Initializing all the Stores at 1731708882611 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708882611Cleaning up temporary data from old regions at 1731708882616 (+5 ms)Running coprocessor post-open hooks at 1731708882619 (+3 ms)Region opened successfully at 1731708882619 2024-11-15T22:14:42,621 WARN [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,qqq,1731708873681.06b9df8f55d791077e9f7ec55ef6a971., aborting... org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server e611192d6313,41231,1731708869041 aborting at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkOpen(RSRpcServices.java:1463) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2235) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:42,621 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2435): Abort already in progress. Ignoring the current request with reason: Failed to open region test,qqq,1731708873681.06b9df8f55d791077e9f7ec55ef6a971. and can not recover 2024-11-15T22:14:42,621 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(132): Open test,www,1731708873681.2e79a8ce6f9715045d056d3dfdef5cec. 2024-11-15T22:14:42,621 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7752): Opening region: {ENCODED => 2e79a8ce6f9715045d056d3dfdef5cec, NAME => 'test,www,1731708873681.2e79a8ce6f9715045d056d3dfdef5cec.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-11-15T22:14:42,621 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 2e79a8ce6f9715045d056d3dfdef5cec 2024-11-15T22:14:42,622 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(898): Instantiated test,www,1731708873681.2e79a8ce6f9715045d056d3dfdef5cec.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:42,622 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7794): checking encryption for 2e79a8ce6f9715045d056d3dfdef5cec 2024-11-15T22:14:42,622 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7797): checking classloading for 2e79a8ce6f9715045d056d3dfdef5cec 2024-11-15T22:14:42,623 INFO [StoreOpener-2e79a8ce6f9715045d056d3dfdef5cec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 2e79a8ce6f9715045d056d3dfdef5cec 2024-11-15T22:14:42,624 INFO [StoreOpener-2e79a8ce6f9715045d056d3dfdef5cec-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2e79a8ce6f9715045d056d3dfdef5cec columnFamilyName family 2024-11-15T22:14:42,624 DEBUG [StoreOpener-2e79a8ce6f9715045d056d3dfdef5cec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:42,625 INFO [StoreOpener-2e79a8ce6f9715045d056d3dfdef5cec-1 {}] regionserver.HStore(327): Store=2e79a8ce6f9715045d056d3dfdef5cec/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:14:42,625 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1038): replaying wal for 2e79a8ce6f9715045d056d3dfdef5cec 2024-11-15T22:14:42,626 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/2e79a8ce6f9715045d056d3dfdef5cec 2024-11-15T22:14:42,627 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/2e79a8ce6f9715045d056d3dfdef5cec 2024-11-15T22:14:42,628 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1048): stopping wal replay for 2e79a8ce6f9715045d056d3dfdef5cec 2024-11-15T22:14:42,628 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1060): Cleaning up temporary data for 2e79a8ce6f9715045d056d3dfdef5cec 2024-11-15T22:14:42,630 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1093): writing seq id for 2e79a8ce6f9715045d056d3dfdef5cec 2024-11-15T22:14:42,631 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1114): Opened 2e79a8ce6f9715045d056d3dfdef5cec; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66366698, jitterRate=-0.01105913519859314}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T22:14:42,631 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2e79a8ce6f9715045d056d3dfdef5cec 2024-11-15T22:14:42,631 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1006): Region open journal for 2e79a8ce6f9715045d056d3dfdef5cec: Running coprocessor pre-open hook at 1731708882622Writing region info on filesystem at 1731708882622Initializing all the Stores at 1731708882623 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708882623Cleaning up temporary data from old regions at 1731708882628 (+5 ms)Running coprocessor post-open hooks at 1731708882631 (+3 ms)Region opened successfully at 1731708882631 2024-11-15T22:14:42,632 WARN [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,www,1731708873681.2e79a8ce6f9715045d056d3dfdef5cec., aborting... org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server e611192d6313,41231,1731708869041 aborting at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkOpen(RSRpcServices.java:1463) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2235) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:42,632 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2435): Abort already in progress. Ignoring the current request with reason: Failed to open region test,www,1731708873681.2e79a8ce6f9715045d056d3dfdef5cec. and can not recover 2024-11-15T22:14:42,632 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] handler.AssignRegionHandler(132): Open test,bbb,1731708873681.897a7bb57a002d365489ce583eb7f84f. 2024-11-15T22:14:42,633 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7752): Opening region: {ENCODED => 897a7bb57a002d365489ce583eb7f84f, NAME => 'test,bbb,1731708873681.897a7bb57a002d365489ce583eb7f84f.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-11-15T22:14:42,633 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 897a7bb57a002d365489ce583eb7f84f 2024-11-15T22:14:42,633 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(898): Instantiated test,bbb,1731708873681.897a7bb57a002d365489ce583eb7f84f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T22:14:42,633 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7794): checking encryption for 897a7bb57a002d365489ce583eb7f84f 2024-11-15T22:14:42,633 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7797): checking classloading for 897a7bb57a002d365489ce583eb7f84f 2024-11-15T22:14:42,635 INFO [StoreOpener-897a7bb57a002d365489ce583eb7f84f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 897a7bb57a002d365489ce583eb7f84f 2024-11-15T22:14:42,636 INFO [StoreOpener-897a7bb57a002d365489ce583eb7f84f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 897a7bb57a002d365489ce583eb7f84f columnFamilyName family 2024-11-15T22:14:42,636 DEBUG [StoreOpener-897a7bb57a002d365489ce583eb7f84f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T22:14:42,637 INFO [StoreOpener-897a7bb57a002d365489ce583eb7f84f-1 {}] regionserver.HStore(327): Store=897a7bb57a002d365489ce583eb7f84f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T22:14:42,637 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1038): replaying wal for 897a7bb57a002d365489ce583eb7f84f 2024-11-15T22:14:42,638 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/897a7bb57a002d365489ce583eb7f84f 2024-11-15T22:14:42,639 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/data/default/test/897a7bb57a002d365489ce583eb7f84f 2024-11-15T22:14:42,639 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1048): stopping wal replay for 897a7bb57a002d365489ce583eb7f84f 2024-11-15T22:14:42,640 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1060): Cleaning up temporary data for 897a7bb57a002d365489ce583eb7f84f 2024-11-15T22:14:42,642 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1093): writing seq id for 897a7bb57a002d365489ce583eb7f84f 2024-11-15T22:14:42,643 INFO [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1114): Opened 897a7bb57a002d365489ce583eb7f84f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63238735, jitterRate=-0.057669416069984436}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-15T22:14:42,643 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 897a7bb57a002d365489ce583eb7f84f 2024-11-15T22:14:42,643 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1006): Region open journal for 897a7bb57a002d365489ce583eb7f84f: Running coprocessor pre-open hook at 1731708882633Writing region info on filesystem at 1731708882633Initializing all the Stores at 1731708882634 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731708882634Cleaning up temporary data from old regions at 1731708882640 (+6 ms)Running coprocessor post-open hooks at 1731708882643 (+3 ms)Region opened successfully at 1731708882643 2024-11-15T22:14:42,644 WARN [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,bbb,1731708873681.897a7bb57a002d365489ce583eb7f84f., aborting... org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server e611192d6313,41231,1731708869041 aborting at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkOpen(RSRpcServices.java:1463) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2235) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T22:14:42,644 DEBUG [RS_OPEN_REGION-regionserver/e611192d6313:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegionServer(2435): Abort already in progress. Ignoring the current request with reason: Failed to open region test,bbb,1731708873681.897a7bb57a002d365489ce583eb7f84f. and can not recover 2024-11-15T22:14:42,650 INFO [RS:1;e611192d6313:41231 {}] regionserver.HRegionServer(976): stopping server e611192d6313,41231,1731708869041; all regions closed. 2024-11-15T22:14:42,651 INFO [RS:0;e611192d6313:38729 {}] regionserver.HRegionServer(976): stopping server e611192d6313,38729,1731708868111; all regions closed. 2024-11-15T22:14:42,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39269 is added to blk_1073741835_1011 (size=5998) 2024-11-15T22:14:42,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39269 is added to blk_1073741865_1041 (size=4192) 2024-11-15T22:14:42,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741865_1041 (size=4192) 2024-11-15T22:14:42,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33631 is added to blk_1073741865_1041 (size=4192) 2024-11-15T22:14:42,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741835_1011 (size=5998) 2024-11-15T22:14:42,658 DEBUG [RS:0;e611192d6313:38729 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/oldWALs 2024-11-15T22:14:42,658 INFO [RS:0;e611192d6313:38729 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL e611192d6313%2C38729%2C1731708868111.meta:.meta(num 1731708880572) 2024-11-15T22:14:42,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33631 is added to blk_1073741834_1010 (size=7064) 2024-11-15T22:14:43,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39269 is added to blk_1073741834_1010 (size=7064) 2024-11-15T22:14:43,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33631 is added to blk_1073741835_1011 (size=5998) 2024-11-15T22:14:43,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741834_1010 (size=7064) 2024-11-15T22:14:43,360 DEBUG [RS:1;e611192d6313:41231 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:14:43,360 INFO [RS:1;e611192d6313:41231 {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T22:14:43,361 INFO [RS:1;e611192d6313:41231 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-15T22:14:43,361 INFO [RS:1;e611192d6313:41231 {}] hbase.ChoreService(370): Chore service for: regionserver/e611192d6313:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-15T22:14:43,362 INFO [RS:1;e611192d6313:41231 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-15T22:14:43,362 INFO [regionserver/e611192d6313:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-15T22:14:43,362 INFO [RS:1;e611192d6313:41231 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-15T22:14:43,362 INFO [RS:1;e611192d6313:41231 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-15T22:14:43,362 INFO [RS:1;e611192d6313:41231 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-15T22:14:43,362 INFO [RS:1;e611192d6313:41231 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:41231 2024-11-15T22:14:43,364 DEBUG [RS:0;e611192d6313:38729 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/oldWALs 2024-11-15T22:14:43,364 INFO [RS:0;e611192d6313:38729 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL e611192d6313%2C38729%2C1731708868111:(num 1731708872744) 2024-11-15T22:14:43,364 DEBUG [RS:0;e611192d6313:38729 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T22:14:43,364 INFO [RS:0;e611192d6313:38729 {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T22:14:43,364 INFO [RS:0;e611192d6313:38729 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-15T22:14:43,364 INFO [RS:0;e611192d6313:38729 {}] hbase.ChoreService(370): Chore service for: regionserver/e611192d6313:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-15T22:14:43,365 INFO [RS:0;e611192d6313:38729 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-15T22:14:43,365 INFO [RS:0;e611192d6313:38729 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:38729 2024-11-15T22:14:43,365 INFO [regionserver/e611192d6313:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-15T22:14:43,611 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39087-0x101409049d30000, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-15T22:14:43,611 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41231-0x101409049d30002, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/e611192d6313,41231,1731708869041 2024-11-15T22:14:43,611 INFO [RS:1;e611192d6313:41231 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-15T22:14:43,753 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x101409049d30001, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/e611192d6313,38729,1731708868111 2024-11-15T22:14:43,753 INFO [RS:0;e611192d6313:38729 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-15T22:14:43,828 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [e611192d6313,38729,1731708868111] 2024-11-15T22:14:43,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41231-0x101409049d30002, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T22:14:43,929 INFO [RS:1;e611192d6313:41231 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-15T22:14:43,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41231-0x101409049d30002, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T22:14:43,929 INFO [RS:1;e611192d6313:41231 {}] regionserver.HRegionServer(1031): Exiting; stopping=e611192d6313,41231,1731708869041; zookeeper connection closed. 2024-11-15T22:14:43,929 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@220917e4 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@220917e4 2024-11-15T22:14:43,947 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/e611192d6313,38729,1731708868111 already deleted, retry=false 2024-11-15T22:14:43,947 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; e611192d6313,38729,1731708868111 expired; onlineServers=1 2024-11-15T22:14:43,947 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [e611192d6313,41231,1731708869041] 2024-11-15T22:14:43,957 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/e611192d6313,41231,1731708869041 already deleted, retry=false 2024-11-15T22:14:43,957 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; e611192d6313,41231,1731708869041 expired; onlineServers=0 2024-11-15T22:14:43,958 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'e611192d6313,39087,1731708867142' ***** 2024-11-15T22:14:43,958 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-15T22:14:43,958 INFO [M:0;e611192d6313:39087 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-15T22:14:43,958 INFO [M:0;e611192d6313:39087 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-15T22:14:43,958 DEBUG [M:0;e611192d6313:39087 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-15T22:14:43,959 DEBUG [M:0;e611192d6313:39087 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-15T22:14:43,959 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-15T22:14:43,959 DEBUG [master/e611192d6313:0:becomeActiveMaster-HFileCleaner.large.0-1731708872261 {}] cleaner.HFileCleaner(306): Exit Thread[master/e611192d6313:0:becomeActiveMaster-HFileCleaner.large.0-1731708872261,5,FailOnTimeoutGroup] 2024-11-15T22:14:43,959 DEBUG [master/e611192d6313:0:becomeActiveMaster-HFileCleaner.small.0-1731708872265 {}] cleaner.HFileCleaner(306): Exit Thread[master/e611192d6313:0:becomeActiveMaster-HFileCleaner.small.0-1731708872265,5,FailOnTimeoutGroup] 2024-11-15T22:14:43,959 INFO [M:0;e611192d6313:39087 {}] hbase.ChoreService(370): Chore service for: master/e611192d6313:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-15T22:14:43,959 INFO [M:0;e611192d6313:39087 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-15T22:14:43,959 DEBUG [M:0;e611192d6313:39087 {}] master.HMaster(1795): Stopping service threads 2024-11-15T22:14:43,959 INFO [M:0;e611192d6313:39087 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-15T22:14:43,960 INFO [M:0;e611192d6313:39087 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T22:14:43,960 INFO [M:0;e611192d6313:39087 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T22:14:43,960 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-15T22:14:43,968 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39087-0x101409049d30000, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-15T22:14:43,968 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39087-0x101409049d30000, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T22:14:43,968 DEBUG [M:0;e611192d6313:39087 {}] zookeeper.ZKUtil(347): master:39087-0x101409049d30000, quorum=127.0.0.1:60621, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-15T22:14:43,969 WARN [M:0;e611192d6313:39087 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-15T22:14:43,970 INFO [M:0;e611192d6313:39087 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/.lastflushedseqids 2024-11-15T22:14:43,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741870_1046 (size=1) 2024-11-15T22:14:43,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39269 is added to blk_1073741870_1046 (size=1) 2024-11-15T22:14:43,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33631 is added to blk_1073741870_1046 (size=1) 2024-11-15T22:14:43,983 INFO [M:0;e611192d6313:39087 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T22:14:43,983 INFO [M:0;e611192d6313:39087 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-15T22:14:43,983 DEBUG [M:0;e611192d6313:39087 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-15T22:14:43,983 INFO [M:0;e611192d6313:39087 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T22:14:43,983 DEBUG [M:0;e611192d6313:39087 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T22:14:43,983 DEBUG [M:0;e611192d6313:39087 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-15T22:14:43,983 DEBUG [M:0;e611192d6313:39087 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T22:14:43,983 INFO [M:0;e611192d6313:39087 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=259.50 KB heapSize=315.42 KB 2024-11-15T22:14:43,994 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x101409049d30001, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T22:14:43,994 INFO [RS:0;e611192d6313:38729 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-15T22:14:43,994 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38729-0x101409049d30001, quorum=127.0.0.1:60621, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T22:14:43,994 INFO [RS:0;e611192d6313:38729 {}] regionserver.HRegionServer(1031): Exiting; stopping=e611192d6313,38729,1731708868111; zookeeper connection closed. 2024-11-15T22:14:43,995 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4d8a972d {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4d8a972d 2024-11-15T22:14:43,995 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-15T22:14:44,003 DEBUG [M:0;e611192d6313:39087 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e4e826e0b95c4ed3abc88ef38a6b9788 is 82, key is hbase:meta,,1/info:regioninfo/1731708880875/Put/seqid=0 2024-11-15T22:14:44,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40641 is added to blk_1073741871_1047 (size=6071) 2024-11-15T22:14:44,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33631 is added to blk_1073741871_1047 (size=6071) 2024-11-15T22:14:44,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39269 is added to blk_1073741871_1047 (size=6071) 2024-11-15T22:14:44,010 INFO [M:0;e611192d6313:39087 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=599 (bloomFilter=true), to=hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e4e826e0b95c4ed3abc88ef38a6b9788 2024-11-15T22:14:44,035 DEBUG [M:0;e611192d6313:39087 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35755/user/jenkins/test-data/8211b3f8-9128-5113-028f-6910e04ededa/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/414ac3d4ffae4606b29c7cae987dd0ed is 2987, key is \x00\x00\x00\x00\x00\x00\x00