2024-11-14 12:54:19,225 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-14 12:54:19,241 main DEBUG Took 0.013227 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-14 12:54:19,242 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-14 12:54:19,242 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-14 12:54:19,243 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-14 12:54:19,245 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 12:54:19,255 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-14 12:54:19,269 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 12:54:19,271 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 12:54:19,271 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 12:54:19,272 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 12:54:19,273 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 12:54:19,273 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 12:54:19,274 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 12:54:19,275 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 12:54:19,275 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 12:54:19,276 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 12:54:19,276 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 12:54:19,277 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 12:54:19,277 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 12:54:19,278 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 12:54:19,278 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 12:54:19,278 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 12:54:19,279 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 12:54:19,279 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 12:54:19,280 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 12:54:19,280 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 12:54:19,280 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 12:54:19,281 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 12:54:19,281 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 12:54:19,281 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 12:54:19,282 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 12:54:19,282 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-14 12:54:19,284 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 12:54:19,285 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-14 12:54:19,286 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-14 12:54:19,287 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-14 12:54:19,288 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-14 12:54:19,288 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-14 12:54:19,296 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-14 12:54:19,299 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-14 12:54:19,302 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-14 12:54:19,302 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-14 12:54:19,302 main DEBUG createAppenders(={Console}) 2024-11-14 12:54:19,303 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 initialized 2024-11-14 12:54:19,304 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-14 12:54:19,304 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 OK. 2024-11-14 12:54:19,304 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-14 12:54:19,305 main DEBUG OutputStream closed 2024-11-14 12:54:19,305 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-14 12:54:19,305 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-14 12:54:19,306 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@c88a337 OK 2024-11-14 12:54:19,380 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-14 12:54:19,383 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-14 12:54:19,385 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-14 12:54:19,386 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-14 12:54:19,386 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-14 12:54:19,387 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-14 12:54:19,387 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-14 12:54:19,388 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-14 12:54:19,388 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-14 12:54:19,389 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-14 12:54:19,389 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-14 12:54:19,390 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-14 12:54:19,390 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-14 12:54:19,391 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-14 12:54:19,391 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-14 12:54:19,391 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-14 12:54:19,392 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-14 12:54:19,392 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-14 12:54:19,395 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-14 12:54:19,395 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@13330ac6) with optional ClassLoader: null 2024-11-14 12:54:19,395 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-14 12:54:19,396 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@13330ac6] started OK. 2024-11-14T12:54:19,409 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP timeout: 13 mins 2024-11-14 12:54:19,412 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-14 12:54:19,412 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-14T12:54:19,643 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20f3381c-3314-c356-689b-6edd39956758 2024-11-14T12:54:19,660 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$HMasterForTest, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-14T12:54:19,680 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20f3381c-3314-c356-689b-6edd39956758/cluster_667a265b-c09c-ca0d-2e08-83a65d1b4d52, deleteOnExit=true 2024-11-14T12:54:19,680 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-14T12:54:19,682 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20f3381c-3314-c356-689b-6edd39956758/test.cache.data in system properties and HBase conf 2024-11-14T12:54:19,683 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20f3381c-3314-c356-689b-6edd39956758/hadoop.tmp.dir in system properties and HBase conf 2024-11-14T12:54:19,683 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20f3381c-3314-c356-689b-6edd39956758/hadoop.log.dir in system properties and HBase conf 2024-11-14T12:54:19,684 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20f3381c-3314-c356-689b-6edd39956758/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-14T12:54:19,685 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20f3381c-3314-c356-689b-6edd39956758/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-14T12:54:19,686 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-14T12:54:19,777 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-14T12:54:19,865 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T12:54:19,869 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20f3381c-3314-c356-689b-6edd39956758/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-14T12:54:19,870 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20f3381c-3314-c356-689b-6edd39956758/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-14T12:54:19,871 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20f3381c-3314-c356-689b-6edd39956758/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-14T12:54:19,871 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20f3381c-3314-c356-689b-6edd39956758/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-14T12:54:19,872 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20f3381c-3314-c356-689b-6edd39956758/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-14T12:54:19,872 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20f3381c-3314-c356-689b-6edd39956758/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-14T12:54:19,873 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20f3381c-3314-c356-689b-6edd39956758/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-14T12:54:19,873 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20f3381c-3314-c356-689b-6edd39956758/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-14T12:54:19,874 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20f3381c-3314-c356-689b-6edd39956758/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-14T12:54:19,874 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20f3381c-3314-c356-689b-6edd39956758/nfs.dump.dir in system properties and HBase conf 2024-11-14T12:54:19,875 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20f3381c-3314-c356-689b-6edd39956758/java.io.tmpdir in system properties and HBase conf 2024-11-14T12:54:19,875 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20f3381c-3314-c356-689b-6edd39956758/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-14T12:54:19,876 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20f3381c-3314-c356-689b-6edd39956758/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-14T12:54:19,877 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20f3381c-3314-c356-689b-6edd39956758/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-14T12:54:20,805 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-14T12:54:20,882 INFO [Time-limited test {}] log.Log(170): Logging initialized @2302ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-14T12:54:20,952 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T12:54:21,012 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T12:54:21,032 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T12:54:21,032 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T12:54:21,034 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-14T12:54:21,045 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T12:54:21,048 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@265b24f1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20f3381c-3314-c356-689b-6edd39956758/hadoop.log.dir/,AVAILABLE} 2024-11-14T12:54:21,049 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@76beb053{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T12:54:21,221 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@376fa8f3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20f3381c-3314-c356-689b-6edd39956758/java.io.tmpdir/jetty-localhost-37333-hadoop-hdfs-3_4_1-tests_jar-_-any-6274967884061364233/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-14T12:54:21,227 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@672b88ea{HTTP/1.1, (http/1.1)}{localhost:37333} 2024-11-14T12:54:21,227 INFO [Time-limited test {}] server.Server(415): Started @2648ms 2024-11-14T12:54:21,698 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T12:54:21,705 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T12:54:21,706 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T12:54:21,707 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T12:54:21,707 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-14T12:54:21,708 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@739a104e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20f3381c-3314-c356-689b-6edd39956758/hadoop.log.dir/,AVAILABLE} 2024-11-14T12:54:21,708 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1c5d2af1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T12:54:21,806 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2226d6a6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20f3381c-3314-c356-689b-6edd39956758/java.io.tmpdir/jetty-localhost-35569-hadoop-hdfs-3_4_1-tests_jar-_-any-7724158484260456849/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T12:54:21,807 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1da1f773{HTTP/1.1, (http/1.1)}{localhost:35569} 2024-11-14T12:54:21,807 INFO [Time-limited test {}] server.Server(415): Started @3228ms 2024-11-14T12:54:21,854 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-14T12:54:21,958 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T12:54:21,963 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T12:54:21,964 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T12:54:21,964 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T12:54:21,965 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-14T12:54:21,966 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@34a3e1ff{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20f3381c-3314-c356-689b-6edd39956758/hadoop.log.dir/,AVAILABLE} 2024-11-14T12:54:21,967 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@12f16529{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T12:54:22,069 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7cfccf27{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20f3381c-3314-c356-689b-6edd39956758/java.io.tmpdir/jetty-localhost-43661-hadoop-hdfs-3_4_1-tests_jar-_-any-2590580172156050912/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T12:54:22,069 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6c41bbb7{HTTP/1.1, (http/1.1)}{localhost:43661} 2024-11-14T12:54:22,070 INFO [Time-limited test {}] server.Server(415): Started @3491ms 2024-11-14T12:54:22,072 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-14T12:54:22,111 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T12:54:22,117 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T12:54:22,119 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T12:54:22,119 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T12:54:22,120 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-14T12:54:22,121 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@11ebaa12{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20f3381c-3314-c356-689b-6edd39956758/hadoop.log.dir/,AVAILABLE} 2024-11-14T12:54:22,122 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@52cf89e6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T12:54:22,231 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@67ddfd25{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20f3381c-3314-c356-689b-6edd39956758/java.io.tmpdir/jetty-localhost-38659-hadoop-hdfs-3_4_1-tests_jar-_-any-603772607151477179/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T12:54:22,232 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@30e38263{HTTP/1.1, (http/1.1)}{localhost:38659} 2024-11-14T12:54:22,232 INFO [Time-limited test {}] server.Server(415): Started @3653ms 2024-11-14T12:54:22,234 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-14T12:54:23,051 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20f3381c-3314-c356-689b-6edd39956758/cluster_667a265b-c09c-ca0d-2e08-83a65d1b4d52/data/data3/current/BP-2037195959-172.17.0.3-1731588860404/current, will proceed with Du for space computation calculation, 2024-11-14T12:54:23,051 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20f3381c-3314-c356-689b-6edd39956758/cluster_667a265b-c09c-ca0d-2e08-83a65d1b4d52/data/data1/current/BP-2037195959-172.17.0.3-1731588860404/current, will proceed with Du for space computation calculation, 2024-11-14T12:54:23,051 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20f3381c-3314-c356-689b-6edd39956758/cluster_667a265b-c09c-ca0d-2e08-83a65d1b4d52/data/data2/current/BP-2037195959-172.17.0.3-1731588860404/current, will proceed with Du for space computation calculation, 2024-11-14T12:54:23,051 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20f3381c-3314-c356-689b-6edd39956758/cluster_667a265b-c09c-ca0d-2e08-83a65d1b4d52/data/data4/current/BP-2037195959-172.17.0.3-1731588860404/current, will proceed with Du for space computation calculation, 2024-11-14T12:54:23,084 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-14T12:54:23,084 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-14T12:54:23,131 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5e78ddf03b5460a6 with lease ID 0xc1cb9af59a38490a: Processing first storage report for DS-9aae07ee-1142-4a11-b0be-2c50c0112df3 from datanode DatanodeRegistration(127.0.0.1:43871, datanodeUuid=e4c4fc83-8f10-4f3a-971b-811762a0334d, infoPort=44851, infoSecurePort=0, ipcPort=33639, storageInfo=lv=-57;cid=testClusterID;nsid=231759395;c=1731588860404) 2024-11-14T12:54:23,132 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5e78ddf03b5460a6 with lease ID 0xc1cb9af59a38490a: from storage DS-9aae07ee-1142-4a11-b0be-2c50c0112df3 node DatanodeRegistration(127.0.0.1:43871, datanodeUuid=e4c4fc83-8f10-4f3a-971b-811762a0334d, infoPort=44851, infoSecurePort=0, ipcPort=33639, storageInfo=lv=-57;cid=testClusterID;nsid=231759395;c=1731588860404), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-14T12:54:23,132 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xacf378de3e649fc7 with lease ID 0xc1cb9af59a384909: Processing first storage report for DS-90f4821d-4e9a-4239-9e98-c91b432844d1 from datanode DatanodeRegistration(127.0.0.1:36021, datanodeUuid=53c1d9b4-1312-4c4c-b470-0154cc93474a, infoPort=38479, infoSecurePort=0, ipcPort=38707, storageInfo=lv=-57;cid=testClusterID;nsid=231759395;c=1731588860404) 2024-11-14T12:54:23,133 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xacf378de3e649fc7 with lease ID 0xc1cb9af59a384909: from storage DS-90f4821d-4e9a-4239-9e98-c91b432844d1 node DatanodeRegistration(127.0.0.1:36021, datanodeUuid=53c1d9b4-1312-4c4c-b470-0154cc93474a, infoPort=38479, infoSecurePort=0, ipcPort=38707, storageInfo=lv=-57;cid=testClusterID;nsid=231759395;c=1731588860404), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T12:54:23,133 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5e78ddf03b5460a6 with lease ID 0xc1cb9af59a38490a: Processing first storage report for DS-a0a86e87-146c-4a85-9304-b8a918253b84 from datanode DatanodeRegistration(127.0.0.1:43871, datanodeUuid=e4c4fc83-8f10-4f3a-971b-811762a0334d, infoPort=44851, infoSecurePort=0, ipcPort=33639, storageInfo=lv=-57;cid=testClusterID;nsid=231759395;c=1731588860404) 2024-11-14T12:54:23,133 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5e78ddf03b5460a6 with lease ID 0xc1cb9af59a38490a: from storage DS-a0a86e87-146c-4a85-9304-b8a918253b84 node DatanodeRegistration(127.0.0.1:43871, datanodeUuid=e4c4fc83-8f10-4f3a-971b-811762a0334d, infoPort=44851, infoSecurePort=0, ipcPort=33639, storageInfo=lv=-57;cid=testClusterID;nsid=231759395;c=1731588860404), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T12:54:23,133 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xacf378de3e649fc7 with lease ID 0xc1cb9af59a384909: Processing first storage report for DS-ac93e800-bb3e-4329-9309-f08b00742127 from datanode DatanodeRegistration(127.0.0.1:36021, datanodeUuid=53c1d9b4-1312-4c4c-b470-0154cc93474a, infoPort=38479, infoSecurePort=0, ipcPort=38707, storageInfo=lv=-57;cid=testClusterID;nsid=231759395;c=1731588860404) 2024-11-14T12:54:23,133 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xacf378de3e649fc7 with lease ID 0xc1cb9af59a384909: from storage DS-ac93e800-bb3e-4329-9309-f08b00742127 node DatanodeRegistration(127.0.0.1:36021, datanodeUuid=53c1d9b4-1312-4c4c-b470-0154cc93474a, infoPort=38479, infoSecurePort=0, ipcPort=38707, storageInfo=lv=-57;cid=testClusterID;nsid=231759395;c=1731588860404), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T12:54:23,161 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20f3381c-3314-c356-689b-6edd39956758/cluster_667a265b-c09c-ca0d-2e08-83a65d1b4d52/data/data6/current/BP-2037195959-172.17.0.3-1731588860404/current, will proceed with Du for space computation calculation, 2024-11-14T12:54:23,161 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20f3381c-3314-c356-689b-6edd39956758/cluster_667a265b-c09c-ca0d-2e08-83a65d1b4d52/data/data5/current/BP-2037195959-172.17.0.3-1731588860404/current, will proceed with Du for space computation calculation, 2024-11-14T12:54:23,183 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-14T12:54:23,188 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7c6788c00c9fa17 with lease ID 0xc1cb9af59a38490b: Processing first storage report for DS-111c8733-6272-4257-b572-55835d3a523b from datanode DatanodeRegistration(127.0.0.1:35109, datanodeUuid=e3951565-5c12-4a17-b84e-9111ca3a7127, infoPort=39561, infoSecurePort=0, ipcPort=43875, storageInfo=lv=-57;cid=testClusterID;nsid=231759395;c=1731588860404) 2024-11-14T12:54:23,188 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7c6788c00c9fa17 with lease ID 0xc1cb9af59a38490b: from storage DS-111c8733-6272-4257-b572-55835d3a523b node DatanodeRegistration(127.0.0.1:35109, datanodeUuid=e3951565-5c12-4a17-b84e-9111ca3a7127, infoPort=39561, infoSecurePort=0, ipcPort=43875, storageInfo=lv=-57;cid=testClusterID;nsid=231759395;c=1731588860404), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T12:54:23,188 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7c6788c00c9fa17 with lease ID 0xc1cb9af59a38490b: Processing first storage report for DS-7b5237e3-1e36-46f7-a06e-478517a66529 from datanode DatanodeRegistration(127.0.0.1:35109, datanodeUuid=e3951565-5c12-4a17-b84e-9111ca3a7127, infoPort=39561, infoSecurePort=0, ipcPort=43875, storageInfo=lv=-57;cid=testClusterID;nsid=231759395;c=1731588860404) 2024-11-14T12:54:23,188 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7c6788c00c9fa17 with lease ID 0xc1cb9af59a38490b: from storage DS-7b5237e3-1e36-46f7-a06e-478517a66529 node DatanodeRegistration(127.0.0.1:35109, datanodeUuid=e3951565-5c12-4a17-b84e-9111ca3a7127, infoPort=39561, infoSecurePort=0, ipcPort=43875, storageInfo=lv=-57;cid=testClusterID;nsid=231759395;c=1731588860404), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T12:54:23,248 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20f3381c-3314-c356-689b-6edd39956758 2024-11-14T12:54:23,320 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20f3381c-3314-c356-689b-6edd39956758/cluster_667a265b-c09c-ca0d-2e08-83a65d1b4d52/zookeeper_0, clientPort=60896, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20f3381c-3314-c356-689b-6edd39956758/cluster_667a265b-c09c-ca0d-2e08-83a65d1b4d52/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/20f3381c-3314-c356-689b-6edd39956758/cluster_667a265b-c09c-ca0d-2e08-83a65d1b4d52/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-14T12:54:23,329 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60896 2024-11-14T12:54:23,338 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T12:54:23,341 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T12:54:23,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36021 is added to blk_1073741825_1001 (size=7) 2024-11-14T12:54:23,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43871 is added to blk_1073741825_1001 (size=7) 2024-11-14T12:54:23,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741825_1001 (size=7) 2024-11-14T12:54:24,049 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625 with version=8 2024-11-14T12:54:24,049 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/hbase-staging 2024-11-14T12:54:24,132 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-14T12:54:24,345 INFO [Time-limited test {}] client.ConnectionUtils(128): master/669ef94e8b26:0 server-side Connection retries=45 2024-11-14T12:54:24,354 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T12:54:24,355 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T12:54:24,359 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T12:54:24,360 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T12:54:24,360 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T12:54:24,489 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-14T12:54:24,546 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-14T12:54:24,557 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-14T12:54:24,562 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T12:54:24,589 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 126021 (auto-detected) 2024-11-14T12:54:24,590 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-14T12:54:24,607 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:45309 2024-11-14T12:54:24,636 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:45309 connecting to ZooKeeper ensemble=127.0.0.1:60896 2024-11-14T12:54:24,757 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:453090x0, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T12:54:24,761 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:45309-0x1013ab2e72b0000 connected 2024-11-14T12:54:24,857 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T12:54:24,860 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T12:54:24,873 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45309-0x1013ab2e72b0000, quorum=127.0.0.1:60896, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T12:54:24,877 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625, hbase.cluster.distributed=false 2024-11-14T12:54:24,904 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45309-0x1013ab2e72b0000, quorum=127.0.0.1:60896, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T12:54:24,909 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45309 2024-11-14T12:54:24,909 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45309 2024-11-14T12:54:24,909 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45309 2024-11-14T12:54:24,910 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45309 2024-11-14T12:54:24,910 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45309 2024-11-14T12:54:25,018 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/669ef94e8b26:0 server-side Connection retries=45 2024-11-14T12:54:25,020 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T12:54:25,020 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T12:54:25,021 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T12:54:25,021 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T12:54:25,021 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T12:54:25,025 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-14T12:54:25,029 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T12:54:25,030 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:43475 2024-11-14T12:54:25,032 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43475 connecting to ZooKeeper ensemble=127.0.0.1:60896 2024-11-14T12:54:25,033 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T12:54:25,040 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T12:54:25,056 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:434750x0, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T12:54:25,057 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43475-0x1013ab2e72b0001 connected 2024-11-14T12:54:25,057 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43475-0x1013ab2e72b0001, quorum=127.0.0.1:60896, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T12:54:25,063 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-14T12:54:25,073 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-14T12:54:25,076 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43475-0x1013ab2e72b0001, quorum=127.0.0.1:60896, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-14T12:54:25,082 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43475-0x1013ab2e72b0001, quorum=127.0.0.1:60896, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T12:54:25,083 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43475 2024-11-14T12:54:25,083 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43475 2024-11-14T12:54:25,083 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43475 2024-11-14T12:54:25,088 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43475 2024-11-14T12:54:25,091 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43475 2024-11-14T12:54:25,107 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/669ef94e8b26:0 server-side Connection retries=45 2024-11-14T12:54:25,108 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T12:54:25,108 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T12:54:25,109 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T12:54:25,109 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T12:54:25,109 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T12:54:25,109 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-14T12:54:25,109 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T12:54:25,111 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:46877 2024-11-14T12:54:25,113 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46877 connecting to ZooKeeper ensemble=127.0.0.1:60896 2024-11-14T12:54:25,114 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T12:54:25,120 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T12:54:25,139 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:468770x0, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T12:54:25,142 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:468770x0, quorum=127.0.0.1:60896, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T12:54:25,142 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-14T12:54:25,143 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46877-0x1013ab2e72b0002 connected 2024-11-14T12:54:25,144 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-14T12:54:25,146 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46877-0x1013ab2e72b0002, quorum=127.0.0.1:60896, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-14T12:54:25,149 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46877-0x1013ab2e72b0002, quorum=127.0.0.1:60896, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T12:54:25,150 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46877 2024-11-14T12:54:25,151 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46877 2024-11-14T12:54:25,155 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46877 2024-11-14T12:54:25,156 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46877 2024-11-14T12:54:25,156 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46877 2024-11-14T12:54:25,174 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/669ef94e8b26:0 server-side Connection retries=45 2024-11-14T12:54:25,174 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T12:54:25,174 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T12:54:25,174 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T12:54:25,175 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T12:54:25,175 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T12:54:25,175 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-14T12:54:25,176 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T12:54:25,178 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:44819 2024-11-14T12:54:25,181 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44819 connecting to ZooKeeper ensemble=127.0.0.1:60896 2024-11-14T12:54:25,182 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T12:54:25,186 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T12:54:25,197 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:448190x0, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T12:54:25,198 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44819-0x1013ab2e72b0003 connected 2024-11-14T12:54:25,201 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44819-0x1013ab2e72b0003, quorum=127.0.0.1:60896, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T12:54:25,203 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-14T12:54:25,204 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-14T12:54:25,205 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44819-0x1013ab2e72b0003, quorum=127.0.0.1:60896, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-14T12:54:25,207 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44819-0x1013ab2e72b0003, quorum=127.0.0.1:60896, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T12:54:25,211 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44819 2024-11-14T12:54:25,211 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44819 2024-11-14T12:54:25,213 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44819 2024-11-14T12:54:25,214 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44819 2024-11-14T12:54:25,214 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44819 2024-11-14T12:54:25,231 DEBUG [M:0;669ef94e8b26:45309 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;669ef94e8b26:45309 2024-11-14T12:54:25,232 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/669ef94e8b26,45309,1731588864185 2024-11-14T12:54:25,248 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45309-0x1013ab2e72b0000, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T12:54:25,248 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43475-0x1013ab2e72b0001, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T12:54:25,248 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44819-0x1013ab2e72b0003, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T12:54:25,248 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46877-0x1013ab2e72b0002, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T12:54:25,251 DEBUG [master/669ef94e8b26:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45309-0x1013ab2e72b0000, quorum=127.0.0.1:60896, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/669ef94e8b26,45309,1731588864185 2024-11-14T12:54:25,281 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44819-0x1013ab2e72b0003, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-14T12:54:25,281 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46877-0x1013ab2e72b0002, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-14T12:54:25,281 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43475-0x1013ab2e72b0001, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-14T12:54:25,281 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45309-0x1013ab2e72b0000, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T12:54:25,281 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46877-0x1013ab2e72b0002, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T12:54:25,281 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43475-0x1013ab2e72b0001, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T12:54:25,281 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44819-0x1013ab2e72b0003, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T12:54:25,282 DEBUG [master/669ef94e8b26:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45309-0x1013ab2e72b0000, quorum=127.0.0.1:60896, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-14T12:54:25,284 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/669ef94e8b26,45309,1731588864185 from backup master directory 2024-11-14T12:54:25,295 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43475-0x1013ab2e72b0001, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T12:54:25,296 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46877-0x1013ab2e72b0002, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T12:54:25,296 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44819-0x1013ab2e72b0003, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T12:54:25,297 WARN [master/669ef94e8b26:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T12:54:25,297 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=669ef94e8b26,45309,1731588864185 2024-11-14T12:54:25,295 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45309-0x1013ab2e72b0000, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/669ef94e8b26,45309,1731588864185 2024-11-14T12:54:25,297 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45309-0x1013ab2e72b0000, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T12:54:25,300 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-14T12:54:25,302 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-14T12:54:25,372 DEBUG [master/669ef94e8b26:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/hbase.id] with ID: e7c3c559-5520-4497-bea0-7e8e506d06ae 2024-11-14T12:54:25,372 DEBUG [master/669ef94e8b26:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/.tmp/hbase.id 2024-11-14T12:54:25,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43871 is added to blk_1073741826_1002 (size=42) 2024-11-14T12:54:25,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36021 is added to blk_1073741826_1002 (size=42) 2024-11-14T12:54:25,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741826_1002 (size=42) 2024-11-14T12:54:25,401 DEBUG [master/669ef94e8b26:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/.tmp/hbase.id]:[hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/hbase.id] 2024-11-14T12:54:25,454 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T12:54:25,461 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-14T12:54:25,483 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 19ms. 2024-11-14T12:54:25,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44819-0x1013ab2e72b0003, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T12:54:25,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46877-0x1013ab2e72b0002, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T12:54:25,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45309-0x1013ab2e72b0000, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T12:54:25,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43475-0x1013ab2e72b0001, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T12:54:25,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741827_1003 (size=196) 2024-11-14T12:54:25,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43871 is added to blk_1073741827_1003 (size=196) 2024-11-14T12:54:25,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36021 is added to blk_1073741827_1003 (size=196) 2024-11-14T12:54:25,547 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T12:54:25,551 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-14T12:54:25,576 DEBUG [master/669ef94e8b26:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T12:54:25,580 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T12:54:25,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43871 is added to blk_1073741828_1004 (size=1189) 2024-11-14T12:54:25,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741828_1004 (size=1189) 2024-11-14T12:54:25,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36021 is added to blk_1073741828_1004 (size=1189) 2024-11-14T12:54:25,646 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/MasterData/data/master/store 2024-11-14T12:54:25,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36021 is added to blk_1073741829_1005 (size=34) 2024-11-14T12:54:25,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43871 is added to blk_1073741829_1005 (size=34) 2024-11-14T12:54:25,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741829_1005 (size=34) 2024-11-14T12:54:25,701 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-14T12:54:25,710 DEBUG [master/669ef94e8b26:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:25,712 DEBUG [master/669ef94e8b26:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-14T12:54:25,712 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T12:54:25,713 DEBUG [master/669ef94e8b26:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T12:54:25,714 DEBUG [master/669ef94e8b26:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-14T12:54:25,715 DEBUG [master/669ef94e8b26:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T12:54:25,715 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T12:54:25,716 DEBUG [master/669ef94e8b26:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731588865712Disabling compacts and flushes for region at 1731588865712Disabling writes for close at 1731588865715 (+3 ms)Writing region close event to WAL at 1731588865715Closed at 1731588865715 2024-11-14T12:54:25,719 WARN [master/669ef94e8b26:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/MasterData/data/master/store/.initializing 2024-11-14T12:54:25,720 DEBUG [master/669ef94e8b26:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/MasterData/WALs/669ef94e8b26,45309,1731588864185 2024-11-14T12:54:25,731 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T12:54:25,751 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=669ef94e8b26%2C45309%2C1731588864185, suffix=, logDir=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/MasterData/WALs/669ef94e8b26,45309,1731588864185, archiveDir=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/MasterData/oldWALs, maxLogs=10 2024-11-14T12:54:25,803 DEBUG [master/669ef94e8b26:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/MasterData/WALs/669ef94e8b26,45309,1731588864185/669ef94e8b26%2C45309%2C1731588864185.1731588865758, exclude list is [], retry=0 2024-11-14T12:54:25,835 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36021,DS-90f4821d-4e9a-4239-9e98-c91b432844d1,DISK] 2024-11-14T12:54:25,835 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35109,DS-111c8733-6272-4257-b572-55835d3a523b,DISK] 2024-11-14T12:54:25,836 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43871,DS-9aae07ee-1142-4a11-b0be-2c50c0112df3,DISK] 2024-11-14T12:54:25,839 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-14T12:54:25,894 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/MasterData/WALs/669ef94e8b26,45309,1731588864185/669ef94e8b26%2C45309%2C1731588864185.1731588865758 2024-11-14T12:54:25,895 DEBUG [master/669ef94e8b26:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44851:44851),(127.0.0.1/127.0.0.1:38479:38479),(127.0.0.1/127.0.0.1:39561:39561)] 2024-11-14T12:54:25,895 DEBUG [master/669ef94e8b26:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-14T12:54:25,896 DEBUG [master/669ef94e8b26:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:25,900 DEBUG [master/669ef94e8b26:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T12:54:25,901 DEBUG [master/669ef94e8b26:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T12:54:25,949 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T12:54:25,978 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-14T12:54:25,983 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T12:54:25,986 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T12:54:25,987 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T12:54:25,990 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-14T12:54:25,991 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T12:54:25,992 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T12:54:25,992 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T12:54:25,996 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-14T12:54:25,996 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T12:54:25,998 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T12:54:25,998 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T12:54:26,000 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-14T12:54:26,001 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T12:54:26,002 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T12:54:26,003 DEBUG [master/669ef94e8b26:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T12:54:26,007 DEBUG [master/669ef94e8b26:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-14T12:54:26,009 DEBUG [master/669ef94e8b26:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-14T12:54:26,016 DEBUG [master/669ef94e8b26:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T12:54:26,017 DEBUG [master/669ef94e8b26:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T12:54:26,022 DEBUG [master/669ef94e8b26:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-14T12:54:26,026 DEBUG [master/669ef94e8b26:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T12:54:26,032 DEBUG [master/669ef94e8b26:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T12:54:26,034 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68455324, jitterRate=0.020063817501068115}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-14T12:54:26,043 DEBUG [master/669ef94e8b26:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731588865915Initializing all the Stores at 1731588865919 (+4 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731588865919Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731588865922 (+3 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731588865923 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731588865923Cleaning up temporary data from old regions at 1731588866018 (+95 ms)Region opened successfully at 1731588866043 (+25 ms) 2024-11-14T12:54:26,045 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-14T12:54:26,093 DEBUG [master/669ef94e8b26:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56f9605b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=669ef94e8b26/172.17.0.3:0 2024-11-14T12:54:26,134 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-14T12:54:26,147 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-14T12:54:26,147 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-14T12:54:26,151 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T12:54:26,152 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-14T12:54:26,158 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-14T12:54:26,158 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-14T12:54:26,187 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-14T12:54:26,197 DEBUG [master/669ef94e8b26:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45309-0x1013ab2e72b0000, quorum=127.0.0.1:60896, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-14T12:54:26,222 DEBUG [master/669ef94e8b26:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-14T12:54:26,225 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-14T12:54:26,226 DEBUG [master/669ef94e8b26:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45309-0x1013ab2e72b0000, quorum=127.0.0.1:60896, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-14T12:54:26,237 DEBUG [master/669ef94e8b26:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-14T12:54:26,240 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-14T12:54:26,243 DEBUG [master/669ef94e8b26:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45309-0x1013ab2e72b0000, quorum=127.0.0.1:60896, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-14T12:54:26,254 DEBUG [master/669ef94e8b26:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-14T12:54:26,256 DEBUG [master/669ef94e8b26:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45309-0x1013ab2e72b0000, quorum=127.0.0.1:60896, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-14T12:54:26,264 DEBUG [master/669ef94e8b26:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-14T12:54:26,283 DEBUG [master/669ef94e8b26:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45309-0x1013ab2e72b0000, quorum=127.0.0.1:60896, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-14T12:54:26,289 DEBUG [master/669ef94e8b26:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-14T12:54:26,304 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44819-0x1013ab2e72b0003, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T12:54:26,304 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43475-0x1013ab2e72b0001, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T12:54:26,304 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43475-0x1013ab2e72b0001, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T12:54:26,304 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44819-0x1013ab2e72b0003, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T12:54:26,305 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46877-0x1013ab2e72b0002, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T12:54:26,305 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45309-0x1013ab2e72b0000, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T12:54:26,305 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46877-0x1013ab2e72b0002, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T12:54:26,305 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45309-0x1013ab2e72b0000, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T12:54:26,308 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=669ef94e8b26,45309,1731588864185, sessionid=0x1013ab2e72b0000, setting cluster-up flag (Was=false) 2024-11-14T12:54:26,339 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44819-0x1013ab2e72b0003, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T12:54:26,339 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46877-0x1013ab2e72b0002, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T12:54:26,339 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43475-0x1013ab2e72b0001, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T12:54:26,339 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45309-0x1013ab2e72b0000, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T12:54:26,364 DEBUG [master/669ef94e8b26:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-14T12:54:26,366 DEBUG [master/669ef94e8b26:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=669ef94e8b26,45309,1731588864185 2024-11-14T12:54:26,389 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46877-0x1013ab2e72b0002, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T12:54:26,389 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44819-0x1013ab2e72b0003, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T12:54:26,389 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45309-0x1013ab2e72b0000, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T12:54:26,389 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43475-0x1013ab2e72b0001, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T12:54:26,414 DEBUG [master/669ef94e8b26:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-14T12:54:26,416 DEBUG [master/669ef94e8b26:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=669ef94e8b26,45309,1731588864185 2024-11-14T12:54:26,424 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-14T12:54:26,515 DEBUG [master/669ef94e8b26:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T12:54:26,519 INFO [RS:0;669ef94e8b26:43475 {}] regionserver.HRegionServer(746): ClusterId : e7c3c559-5520-4497-bea0-7e8e506d06ae 2024-11-14T12:54:26,519 INFO [RS:1;669ef94e8b26:46877 {}] regionserver.HRegionServer(746): ClusterId : e7c3c559-5520-4497-bea0-7e8e506d06ae 2024-11-14T12:54:26,522 DEBUG [RS:1;669ef94e8b26:46877 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-14T12:54:26,523 DEBUG [RS:0;669ef94e8b26:43475 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-14T12:54:26,523 INFO [RS:2;669ef94e8b26:44819 {}] regionserver.HRegionServer(746): ClusterId : e7c3c559-5520-4497-bea0-7e8e506d06ae 2024-11-14T12:54:26,524 DEBUG [RS:2;669ef94e8b26:44819 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-14T12:54:26,527 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-14T12:54:26,534 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-14T12:54:26,541 DEBUG [RS:1;669ef94e8b26:46877 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-14T12:54:26,541 DEBUG [RS:2;669ef94e8b26:44819 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-14T12:54:26,541 DEBUG [RS:1;669ef94e8b26:46877 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-14T12:54:26,541 DEBUG [RS:2;669ef94e8b26:44819 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-14T12:54:26,542 DEBUG [RS:0;669ef94e8b26:43475 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-14T12:54:26,543 DEBUG [RS:0;669ef94e8b26:43475 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-14T12:54:26,541 DEBUG [master/669ef94e8b26:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 669ef94e8b26,45309,1731588864185 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-14T12:54:26,557 DEBUG [RS:2;669ef94e8b26:44819 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-14T12:54:26,557 DEBUG [RS:1;669ef94e8b26:46877 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-14T12:54:26,557 DEBUG [RS:0;669ef94e8b26:43475 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-14T12:54:26,557 DEBUG [RS:1;669ef94e8b26:46877 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7eb1ce69, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=669ef94e8b26/172.17.0.3:0 2024-11-14T12:54:26,557 DEBUG [RS:0;669ef94e8b26:43475 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e11bc10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=669ef94e8b26/172.17.0.3:0 2024-11-14T12:54:26,557 DEBUG [RS:2;669ef94e8b26:44819 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13c60b4b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=669ef94e8b26/172.17.0.3:0 2024-11-14T12:54:26,558 DEBUG [master/669ef94e8b26:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/669ef94e8b26:0, corePoolSize=5, maxPoolSize=5 2024-11-14T12:54:26,559 DEBUG [master/669ef94e8b26:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/669ef94e8b26:0, corePoolSize=5, maxPoolSize=5 2024-11-14T12:54:26,559 DEBUG [master/669ef94e8b26:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/669ef94e8b26:0, corePoolSize=5, maxPoolSize=5 2024-11-14T12:54:26,559 DEBUG [master/669ef94e8b26:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/669ef94e8b26:0, corePoolSize=5, maxPoolSize=5 2024-11-14T12:54:26,559 DEBUG [master/669ef94e8b26:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/669ef94e8b26:0, corePoolSize=10, maxPoolSize=10 2024-11-14T12:54:26,559 DEBUG [master/669ef94e8b26:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/669ef94e8b26:0, corePoolSize=1, maxPoolSize=1 2024-11-14T12:54:26,559 DEBUG [master/669ef94e8b26:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/669ef94e8b26:0, corePoolSize=2, maxPoolSize=2 2024-11-14T12:54:26,560 DEBUG [master/669ef94e8b26:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/669ef94e8b26:0, corePoolSize=1, maxPoolSize=1 2024-11-14T12:54:26,568 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731588896568 2024-11-14T12:54:26,570 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-14T12:54:26,572 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-14T12:54:26,576 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-14T12:54:26,576 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-14T12:54:26,577 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-14T12:54:26,577 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-14T12:54:26,580 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T12:54:26,581 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-14T12:54:26,585 DEBUG [RS:1;669ef94e8b26:46877 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;669ef94e8b26:46877 2024-11-14T12:54:26,588 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T12:54:26,588 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-14T12:54:26,589 DEBUG [RS:0;669ef94e8b26:43475 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;669ef94e8b26:43475 2024-11-14T12:54:26,589 DEBUG [RS:2;669ef94e8b26:44819 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;669ef94e8b26:44819 2024-11-14T12:54:26,589 INFO [RS:1;669ef94e8b26:46877 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-14T12:54:26,589 INFO [RS:0;669ef94e8b26:43475 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-14T12:54:26,589 INFO [RS:1;669ef94e8b26:46877 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-14T12:54:26,589 INFO [RS:0;669ef94e8b26:43475 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-14T12:54:26,589 INFO [RS:2;669ef94e8b26:44819 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-14T12:54:26,589 DEBUG [RS:0;669ef94e8b26:43475 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-14T12:54:26,589 DEBUG [RS:1;669ef94e8b26:46877 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-14T12:54:26,589 INFO [RS:2;669ef94e8b26:44819 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-14T12:54:26,590 DEBUG [RS:2;669ef94e8b26:44819 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-14T12:54:26,581 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-14T12:54:26,593 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-14T12:54:26,593 INFO [RS:0;669ef94e8b26:43475 {}] regionserver.HRegionServer(2659): reportForDuty to master=669ef94e8b26,45309,1731588864185 with port=43475, startcode=1731588864980 2024-11-14T12:54:26,594 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-14T12:54:26,595 INFO [RS:1;669ef94e8b26:46877 {}] regionserver.HRegionServer(2659): reportForDuty to master=669ef94e8b26,45309,1731588864185 with port=46877, startcode=1731588865107 2024-11-14T12:54:26,595 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-14T12:54:26,595 INFO [RS:2;669ef94e8b26:44819 {}] regionserver.HRegionServer(2659): reportForDuty to master=669ef94e8b26,45309,1731588864185 with port=44819, startcode=1731588865173 2024-11-14T12:54:26,599 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-14T12:54:26,600 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-14T12:54:26,608 DEBUG [RS:1;669ef94e8b26:46877 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-14T12:54:26,608 DEBUG [RS:0;669ef94e8b26:43475 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-14T12:54:26,609 DEBUG [RS:2;669ef94e8b26:44819 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-14T12:54:26,609 DEBUG [master/669ef94e8b26:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/669ef94e8b26:0:becomeActiveMaster-HFileCleaner.large.0-1731588866601,5,FailOnTimeoutGroup] 2024-11-14T12:54:26,612 DEBUG [master/669ef94e8b26:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/669ef94e8b26:0:becomeActiveMaster-HFileCleaner.small.0-1731588866610,5,FailOnTimeoutGroup] 2024-11-14T12:54:26,612 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-14T12:54:26,612 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-14T12:54:26,613 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-14T12:54:26,614 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-14T12:54:26,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43871 is added to blk_1073741831_1007 (size=1321) 2024-11-14T12:54:26,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36021 is added to blk_1073741831_1007 (size=1321) 2024-11-14T12:54:26,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741831_1007 (size=1321) 2024-11-14T12:54:26,630 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-14T12:54:26,631 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625 2024-11-14T12:54:26,662 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34327, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-14T12:54:26,662 INFO [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59875, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-14T12:54:26,662 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36833, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-14T12:54:26,669 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45309 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 669ef94e8b26,44819,1731588865173 2024-11-14T12:54:26,672 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45309 {}] master.ServerManager(517): Registering regionserver=669ef94e8b26,44819,1731588865173 2024-11-14T12:54:26,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36021 is added to blk_1073741832_1008 (size=32) 2024-11-14T12:54:26,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741832_1008 (size=32) 2024-11-14T12:54:26,690 DEBUG [RS:2;669ef94e8b26:44819 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625 2024-11-14T12:54:26,690 DEBUG [RS:2;669ef94e8b26:44819 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36759 2024-11-14T12:54:26,690 DEBUG [RS:2;669ef94e8b26:44819 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-14T12:54:26,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43871 is added to blk_1073741832_1008 (size=32) 2024-11-14T12:54:26,692 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45309 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 669ef94e8b26,46877,1731588865107 2024-11-14T12:54:26,693 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45309 {}] master.ServerManager(517): Registering regionserver=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:26,693 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:26,697 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-14T12:54:26,703 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T12:54:26,703 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T12:54:26,703 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45309 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 669ef94e8b26,43475,1731588864980 2024-11-14T12:54:26,704 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45309 {}] master.ServerManager(517): Registering regionserver=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:26,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45309-0x1013ab2e72b0000, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T12:54:26,704 DEBUG [RS:1;669ef94e8b26:46877 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625 2024-11-14T12:54:26,704 DEBUG [RS:1;669ef94e8b26:46877 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36759 2024-11-14T12:54:26,704 DEBUG [RS:1;669ef94e8b26:46877 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-14T12:54:26,706 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T12:54:26,707 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-14T12:54:26,708 DEBUG [RS:0;669ef94e8b26:43475 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625 2024-11-14T12:54:26,708 DEBUG [RS:0;669ef94e8b26:43475 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36759 2024-11-14T12:54:26,708 DEBUG [RS:0;669ef94e8b26:43475 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-14T12:54:26,711 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T12:54:26,711 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T12:54:26,714 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T12:54:26,715 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-14T12:54:26,719 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T12:54:26,719 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T12:54:26,725 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T12:54:26,725 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-14T12:54:26,733 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T12:54:26,733 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T12:54:26,734 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T12:54:26,735 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T12:54:26,737 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/hbase/meta/1588230740 2024-11-14T12:54:26,738 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/hbase/meta/1588230740 2024-11-14T12:54:26,741 DEBUG [RS:2;669ef94e8b26:44819 {}] zookeeper.ZKUtil(111): regionserver:44819-0x1013ab2e72b0003, quorum=127.0.0.1:60896, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/669ef94e8b26,44819,1731588865173 2024-11-14T12:54:26,741 WARN [RS:2;669ef94e8b26:44819 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T12:54:26,741 INFO [RS:2;669ef94e8b26:44819 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T12:54:26,742 DEBUG [RS:2;669ef94e8b26:44819 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/WALs/669ef94e8b26,44819,1731588865173 2024-11-14T12:54:26,742 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T12:54:26,742 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T12:54:26,743 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-14T12:54:26,746 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T12:54:26,756 DEBUG [RS:1;669ef94e8b26:46877 {}] zookeeper.ZKUtil(111): regionserver:46877-0x1013ab2e72b0002, quorum=127.0.0.1:60896, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/669ef94e8b26,46877,1731588865107 2024-11-14T12:54:26,756 WARN [RS:1;669ef94e8b26:46877 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T12:54:26,756 INFO [RS:1;669ef94e8b26:46877 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T12:54:26,756 DEBUG [RS:1;669ef94e8b26:46877 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/WALs/669ef94e8b26,46877,1731588865107 2024-11-14T12:54:26,757 DEBUG [RS:0;669ef94e8b26:43475 {}] zookeeper.ZKUtil(111): regionserver:43475-0x1013ab2e72b0001, quorum=127.0.0.1:60896, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/669ef94e8b26,43475,1731588864980 2024-11-14T12:54:26,757 WARN [RS:0;669ef94e8b26:43475 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T12:54:26,757 INFO [RS:0;669ef94e8b26:43475 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T12:54:26,757 DEBUG [RS:0;669ef94e8b26:43475 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/WALs/669ef94e8b26,43475,1731588864980 2024-11-14T12:54:26,758 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [669ef94e8b26,43475,1731588864980] 2024-11-14T12:54:26,759 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [669ef94e8b26,46877,1731588865107] 2024-11-14T12:54:26,759 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [669ef94e8b26,44819,1731588865173] 2024-11-14T12:54:26,765 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T12:54:26,772 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62708498, jitterRate=-0.06557056307792664}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-14T12:54:26,777 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731588866693Initializing all the Stores at 1731588866697 (+4 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731588866697Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731588866697Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731588866697Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731588866697Cleaning up temporary data from old regions at 1731588866742 (+45 ms)Region opened successfully at 1731588866777 (+35 ms) 2024-11-14T12:54:26,780 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T12:54:26,780 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T12:54:26,780 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T12:54:26,780 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T12:54:26,780 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T12:54:26,790 INFO [RS:0;669ef94e8b26:43475 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-14T12:54:26,790 INFO [RS:2;669ef94e8b26:44819 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-14T12:54:26,792 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T12:54:26,793 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731588866780Disabling compacts and flushes for region at 1731588866780Disabling writes for close at 1731588866780Writing region close event to WAL at 1731588866792 (+12 ms)Closed at 1731588866792 2024-11-14T12:54:26,798 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T12:54:26,798 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-14T12:54:26,804 INFO [RS:1;669ef94e8b26:46877 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-14T12:54:26,808 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-14T12:54:26,821 INFO [RS:2;669ef94e8b26:44819 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-14T12:54:26,827 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T12:54:26,827 INFO [RS:0;669ef94e8b26:43475 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-14T12:54:26,828 INFO [RS:1;669ef94e8b26:46877 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-14T12:54:26,831 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-14T12:54:26,837 INFO [RS:2;669ef94e8b26:44819 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-14T12:54:26,837 INFO [RS:0;669ef94e8b26:43475 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-14T12:54:26,837 INFO [RS:2;669ef94e8b26:44819 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T12:54:26,837 INFO [RS:0;669ef94e8b26:43475 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T12:54:26,838 INFO [RS:1;669ef94e8b26:46877 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-14T12:54:26,838 INFO [RS:1;669ef94e8b26:46877 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T12:54:26,840 INFO [RS:0;669ef94e8b26:43475 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-14T12:54:26,843 INFO [RS:1;669ef94e8b26:46877 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-14T12:54:26,844 INFO [RS:2;669ef94e8b26:44819 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-14T12:54:26,849 INFO [RS:2;669ef94e8b26:44819 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-14T12:54:26,850 INFO [RS:1;669ef94e8b26:46877 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-14T12:54:26,850 INFO [RS:0;669ef94e8b26:43475 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-14T12:54:26,853 INFO [RS:0;669ef94e8b26:43475 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-14T12:54:26,853 INFO [RS:2;669ef94e8b26:44819 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-14T12:54:26,853 INFO [RS:1;669ef94e8b26:46877 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-14T12:54:26,854 DEBUG [RS:1;669ef94e8b26:46877 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/669ef94e8b26:0, corePoolSize=1, maxPoolSize=1 2024-11-14T12:54:26,854 DEBUG [RS:2;669ef94e8b26:44819 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/669ef94e8b26:0, corePoolSize=1, maxPoolSize=1 2024-11-14T12:54:26,854 DEBUG [RS:0;669ef94e8b26:43475 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/669ef94e8b26:0, corePoolSize=1, maxPoolSize=1 2024-11-14T12:54:26,854 DEBUG [RS:1;669ef94e8b26:46877 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/669ef94e8b26:0, corePoolSize=1, maxPoolSize=1 2024-11-14T12:54:26,854 DEBUG [RS:0;669ef94e8b26:43475 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/669ef94e8b26:0, corePoolSize=1, maxPoolSize=1 2024-11-14T12:54:26,854 DEBUG [RS:2;669ef94e8b26:44819 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/669ef94e8b26:0, corePoolSize=1, maxPoolSize=1 2024-11-14T12:54:26,854 DEBUG [RS:1;669ef94e8b26:46877 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/669ef94e8b26:0, corePoolSize=1, maxPoolSize=1 2024-11-14T12:54:26,854 DEBUG [RS:0;669ef94e8b26:43475 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/669ef94e8b26:0, corePoolSize=1, maxPoolSize=1 2024-11-14T12:54:26,854 DEBUG [RS:1;669ef94e8b26:46877 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/669ef94e8b26:0, corePoolSize=1, maxPoolSize=1 2024-11-14T12:54:26,854 DEBUG [RS:1;669ef94e8b26:46877 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/669ef94e8b26:0, corePoolSize=1, maxPoolSize=1 2024-11-14T12:54:26,854 DEBUG [RS:0;669ef94e8b26:43475 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/669ef94e8b26:0, corePoolSize=1, maxPoolSize=1 2024-11-14T12:54:26,854 DEBUG [RS:2;669ef94e8b26:44819 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/669ef94e8b26:0, corePoolSize=1, maxPoolSize=1 2024-11-14T12:54:26,855 DEBUG [RS:0;669ef94e8b26:43475 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/669ef94e8b26:0, corePoolSize=1, maxPoolSize=1 2024-11-14T12:54:26,855 DEBUG [RS:1;669ef94e8b26:46877 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/669ef94e8b26:0, corePoolSize=2, maxPoolSize=2 2024-11-14T12:54:26,855 DEBUG [RS:2;669ef94e8b26:44819 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/669ef94e8b26:0, corePoolSize=1, maxPoolSize=1 2024-11-14T12:54:26,855 DEBUG [RS:0;669ef94e8b26:43475 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/669ef94e8b26:0, corePoolSize=2, maxPoolSize=2 2024-11-14T12:54:26,855 DEBUG [RS:1;669ef94e8b26:46877 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/669ef94e8b26:0, corePoolSize=1, maxPoolSize=1 2024-11-14T12:54:26,855 DEBUG [RS:2;669ef94e8b26:44819 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/669ef94e8b26:0, corePoolSize=1, maxPoolSize=1 2024-11-14T12:54:26,855 DEBUG [RS:0;669ef94e8b26:43475 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/669ef94e8b26:0, corePoolSize=1, maxPoolSize=1 2024-11-14T12:54:26,855 DEBUG [RS:1;669ef94e8b26:46877 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/669ef94e8b26:0, corePoolSize=1, maxPoolSize=1 2024-11-14T12:54:26,855 DEBUG [RS:2;669ef94e8b26:44819 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/669ef94e8b26:0, corePoolSize=2, maxPoolSize=2 2024-11-14T12:54:26,855 DEBUG [RS:0;669ef94e8b26:43475 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/669ef94e8b26:0, corePoolSize=1, maxPoolSize=1 2024-11-14T12:54:26,855 DEBUG [RS:1;669ef94e8b26:46877 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/669ef94e8b26:0, corePoolSize=1, maxPoolSize=1 2024-11-14T12:54:26,855 DEBUG [RS:2;669ef94e8b26:44819 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/669ef94e8b26:0, corePoolSize=1, maxPoolSize=1 2024-11-14T12:54:26,855 DEBUG [RS:0;669ef94e8b26:43475 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/669ef94e8b26:0, corePoolSize=1, maxPoolSize=1 2024-11-14T12:54:26,855 DEBUG [RS:2;669ef94e8b26:44819 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/669ef94e8b26:0, corePoolSize=1, maxPoolSize=1 2024-11-14T12:54:26,855 DEBUG [RS:1;669ef94e8b26:46877 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/669ef94e8b26:0, corePoolSize=1, maxPoolSize=1 2024-11-14T12:54:26,856 DEBUG [RS:2;669ef94e8b26:44819 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/669ef94e8b26:0, corePoolSize=1, maxPoolSize=1 2024-11-14T12:54:26,856 DEBUG [RS:1;669ef94e8b26:46877 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/669ef94e8b26:0, corePoolSize=1, maxPoolSize=1 2024-11-14T12:54:26,856 DEBUG [RS:2;669ef94e8b26:44819 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/669ef94e8b26:0, corePoolSize=1, maxPoolSize=1 2024-11-14T12:54:26,856 DEBUG [RS:1;669ef94e8b26:46877 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/669ef94e8b26:0, corePoolSize=1, maxPoolSize=1 2024-11-14T12:54:26,856 DEBUG [RS:2;669ef94e8b26:44819 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/669ef94e8b26:0, corePoolSize=1, maxPoolSize=1 2024-11-14T12:54:26,856 DEBUG [RS:2;669ef94e8b26:44819 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/669ef94e8b26:0, corePoolSize=1, maxPoolSize=1 2024-11-14T12:54:26,856 DEBUG [RS:0;669ef94e8b26:43475 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/669ef94e8b26:0, corePoolSize=1, maxPoolSize=1 2024-11-14T12:54:26,859 DEBUG [RS:0;669ef94e8b26:43475 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/669ef94e8b26:0, corePoolSize=1, maxPoolSize=1 2024-11-14T12:54:26,860 DEBUG [RS:0;669ef94e8b26:43475 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/669ef94e8b26:0, corePoolSize=1, maxPoolSize=1 2024-11-14T12:54:26,860 DEBUG [RS:0;669ef94e8b26:43475 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/669ef94e8b26:0, corePoolSize=3, maxPoolSize=3 2024-11-14T12:54:26,860 DEBUG [RS:0;669ef94e8b26:43475 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/669ef94e8b26:0, corePoolSize=3, maxPoolSize=3 2024-11-14T12:54:26,860 DEBUG [RS:2;669ef94e8b26:44819 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/669ef94e8b26:0, corePoolSize=3, maxPoolSize=3 2024-11-14T12:54:26,861 DEBUG [RS:1;669ef94e8b26:46877 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/669ef94e8b26:0, corePoolSize=3, maxPoolSize=3 2024-11-14T12:54:26,861 DEBUG [RS:2;669ef94e8b26:44819 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/669ef94e8b26:0, corePoolSize=3, maxPoolSize=3 2024-11-14T12:54:26,861 DEBUG [RS:1;669ef94e8b26:46877 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/669ef94e8b26:0, corePoolSize=3, maxPoolSize=3 2024-11-14T12:54:26,864 INFO [RS:2;669ef94e8b26:44819 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T12:54:26,864 INFO [RS:2;669ef94e8b26:44819 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T12:54:26,865 INFO [RS:2;669ef94e8b26:44819 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T12:54:26,865 INFO [RS:2;669ef94e8b26:44819 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-14T12:54:26,865 INFO [RS:2;669ef94e8b26:44819 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-14T12:54:26,865 INFO [RS:2;669ef94e8b26:44819 {}] hbase.ChoreService(168): Chore ScheduledChore name=669ef94e8b26,44819,1731588865173-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T12:54:26,874 INFO [RS:0;669ef94e8b26:43475 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T12:54:26,874 INFO [RS:0;669ef94e8b26:43475 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T12:54:26,874 INFO [RS:0;669ef94e8b26:43475 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T12:54:26,874 INFO [RS:0;669ef94e8b26:43475 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-14T12:54:26,874 INFO [RS:0;669ef94e8b26:43475 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-14T12:54:26,874 INFO [RS:0;669ef94e8b26:43475 {}] hbase.ChoreService(168): Chore ScheduledChore name=669ef94e8b26,43475,1731588864980-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T12:54:26,886 INFO [RS:1;669ef94e8b26:46877 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T12:54:26,887 INFO [RS:1;669ef94e8b26:46877 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T12:54:26,888 INFO [RS:1;669ef94e8b26:46877 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T12:54:26,889 INFO [RS:1;669ef94e8b26:46877 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-14T12:54:26,889 INFO [RS:1;669ef94e8b26:46877 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-14T12:54:26,889 INFO [RS:1;669ef94e8b26:46877 {}] hbase.ChoreService(168): Chore ScheduledChore name=669ef94e8b26,46877,1731588865107-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T12:54:26,901 INFO [RS:2;669ef94e8b26:44819 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-14T12:54:26,904 INFO [RS:0;669ef94e8b26:43475 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-14T12:54:26,908 INFO [RS:0;669ef94e8b26:43475 {}] hbase.ChoreService(168): Chore ScheduledChore name=669ef94e8b26,43475,1731588864980-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T12:54:26,908 INFO [RS:0;669ef94e8b26:43475 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T12:54:26,906 INFO [RS:2;669ef94e8b26:44819 {}] hbase.ChoreService(168): Chore ScheduledChore name=669ef94e8b26,44819,1731588865173-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T12:54:26,909 INFO [RS:0;669ef94e8b26:43475 {}] regionserver.Replication(171): 669ef94e8b26,43475,1731588864980 started 2024-11-14T12:54:26,909 INFO [RS:2;669ef94e8b26:44819 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T12:54:26,909 INFO [RS:2;669ef94e8b26:44819 {}] regionserver.Replication(171): 669ef94e8b26,44819,1731588865173 started 2024-11-14T12:54:26,939 INFO [RS:1;669ef94e8b26:46877 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-14T12:54:26,942 INFO [RS:0;669ef94e8b26:43475 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T12:54:26,942 INFO [RS:0;669ef94e8b26:43475 {}] regionserver.HRegionServer(1482): Serving as 669ef94e8b26,43475,1731588864980, RpcServer on 669ef94e8b26/172.17.0.3:43475, sessionid=0x1013ab2e72b0001 2024-11-14T12:54:26,943 INFO [RS:1;669ef94e8b26:46877 {}] hbase.ChoreService(168): Chore ScheduledChore name=669ef94e8b26,46877,1731588865107-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T12:54:26,943 DEBUG [RS:0;669ef94e8b26:43475 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-14T12:54:26,944 DEBUG [RS:0;669ef94e8b26:43475 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 669ef94e8b26,43475,1731588864980 2024-11-14T12:54:26,944 DEBUG [RS:0;669ef94e8b26:43475 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '669ef94e8b26,43475,1731588864980' 2024-11-14T12:54:26,944 DEBUG [RS:0;669ef94e8b26:43475 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-14T12:54:26,944 INFO [RS:1;669ef94e8b26:46877 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T12:54:26,945 INFO [RS:1;669ef94e8b26:46877 {}] regionserver.Replication(171): 669ef94e8b26,46877,1731588865107 started 2024-11-14T12:54:26,950 INFO [RS:2;669ef94e8b26:44819 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T12:54:26,950 INFO [RS:2;669ef94e8b26:44819 {}] regionserver.HRegionServer(1482): Serving as 669ef94e8b26,44819,1731588865173, RpcServer on 669ef94e8b26/172.17.0.3:44819, sessionid=0x1013ab2e72b0003 2024-11-14T12:54:26,950 DEBUG [RS:2;669ef94e8b26:44819 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-14T12:54:26,950 DEBUG [RS:2;669ef94e8b26:44819 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 669ef94e8b26,44819,1731588865173 2024-11-14T12:54:26,951 DEBUG [RS:2;669ef94e8b26:44819 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '669ef94e8b26,44819,1731588865173' 2024-11-14T12:54:26,951 DEBUG [RS:2;669ef94e8b26:44819 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-14T12:54:26,952 DEBUG [RS:0;669ef94e8b26:43475 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-14T12:54:26,952 DEBUG [RS:2;669ef94e8b26:44819 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-14T12:54:26,954 DEBUG [RS:0;669ef94e8b26:43475 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-14T12:54:26,954 DEBUG [RS:0;669ef94e8b26:43475 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-14T12:54:26,955 DEBUG [RS:0;669ef94e8b26:43475 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 669ef94e8b26,43475,1731588864980 2024-11-14T12:54:26,955 DEBUG [RS:0;669ef94e8b26:43475 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '669ef94e8b26,43475,1731588864980' 2024-11-14T12:54:26,955 DEBUG [RS:0;669ef94e8b26:43475 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-14T12:54:26,955 DEBUG [RS:2;669ef94e8b26:44819 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-14T12:54:26,955 DEBUG [RS:2;669ef94e8b26:44819 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-14T12:54:26,955 DEBUG [RS:2;669ef94e8b26:44819 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 669ef94e8b26,44819,1731588865173 2024-11-14T12:54:26,955 DEBUG [RS:2;669ef94e8b26:44819 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '669ef94e8b26,44819,1731588865173' 2024-11-14T12:54:26,955 DEBUG [RS:2;669ef94e8b26:44819 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-14T12:54:26,962 DEBUG [RS:2;669ef94e8b26:44819 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-14T12:54:26,963 DEBUG [RS:2;669ef94e8b26:44819 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-14T12:54:26,963 INFO [RS:2;669ef94e8b26:44819 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-14T12:54:26,963 INFO [RS:2;669ef94e8b26:44819 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-14T12:54:26,960 DEBUG [RS:0;669ef94e8b26:43475 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-14T12:54:26,969 DEBUG [RS:0;669ef94e8b26:43475 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-14T12:54:26,969 INFO [RS:0;669ef94e8b26:43475 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-14T12:54:26,969 INFO [RS:0;669ef94e8b26:43475 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-14T12:54:26,970 INFO [RS:1;669ef94e8b26:46877 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T12:54:26,970 INFO [RS:1;669ef94e8b26:46877 {}] regionserver.HRegionServer(1482): Serving as 669ef94e8b26,46877,1731588865107, RpcServer on 669ef94e8b26/172.17.0.3:46877, sessionid=0x1013ab2e72b0002 2024-11-14T12:54:26,970 DEBUG [RS:1;669ef94e8b26:46877 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-14T12:54:26,971 DEBUG [RS:1;669ef94e8b26:46877 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 669ef94e8b26,46877,1731588865107 2024-11-14T12:54:26,971 DEBUG [RS:1;669ef94e8b26:46877 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '669ef94e8b26,46877,1731588865107' 2024-11-14T12:54:26,971 DEBUG [RS:1;669ef94e8b26:46877 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-14T12:54:26,972 DEBUG [RS:1;669ef94e8b26:46877 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-14T12:54:26,973 DEBUG [RS:1;669ef94e8b26:46877 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-14T12:54:26,973 DEBUG [RS:1;669ef94e8b26:46877 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-14T12:54:26,973 DEBUG [RS:1;669ef94e8b26:46877 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 669ef94e8b26,46877,1731588865107 2024-11-14T12:54:26,973 DEBUG [RS:1;669ef94e8b26:46877 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '669ef94e8b26,46877,1731588865107' 2024-11-14T12:54:26,973 DEBUG [RS:1;669ef94e8b26:46877 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-14T12:54:26,974 DEBUG [RS:1;669ef94e8b26:46877 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-14T12:54:26,975 DEBUG [RS:1;669ef94e8b26:46877 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-14T12:54:26,975 INFO [RS:1;669ef94e8b26:46877 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-14T12:54:26,975 INFO [RS:1;669ef94e8b26:46877 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-14T12:54:26,983 WARN [669ef94e8b26:45309 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-14T12:54:27,070 INFO [RS:2;669ef94e8b26:44819 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T12:54:27,070 INFO [RS:0;669ef94e8b26:43475 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T12:54:27,074 INFO [RS:2;669ef94e8b26:44819 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=669ef94e8b26%2C44819%2C1731588865173, suffix=, logDir=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/WALs/669ef94e8b26,44819,1731588865173, archiveDir=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/oldWALs, maxLogs=32 2024-11-14T12:54:27,074 INFO [RS:0;669ef94e8b26:43475 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=669ef94e8b26%2C43475%2C1731588864980, suffix=, logDir=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/WALs/669ef94e8b26,43475,1731588864980, archiveDir=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/oldWALs, maxLogs=32 2024-11-14T12:54:27,075 INFO [RS:1;669ef94e8b26:46877 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T12:54:27,081 INFO [RS:1;669ef94e8b26:46877 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=669ef94e8b26%2C46877%2C1731588865107, suffix=, logDir=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/WALs/669ef94e8b26,46877,1731588865107, archiveDir=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/oldWALs, maxLogs=32 2024-11-14T12:54:27,095 DEBUG [RS:2;669ef94e8b26:44819 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/WALs/669ef94e8b26,44819,1731588865173/669ef94e8b26%2C44819%2C1731588865173.1731588867078, exclude list is [], retry=0 2024-11-14T12:54:27,096 DEBUG [RS:0;669ef94e8b26:43475 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/WALs/669ef94e8b26,43475,1731588864980/669ef94e8b26%2C43475%2C1731588864980.1731588867077, exclude list is [], retry=0 2024-11-14T12:54:27,099 DEBUG [RS:1;669ef94e8b26:46877 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/WALs/669ef94e8b26,46877,1731588865107/669ef94e8b26%2C46877%2C1731588865107.1731588867083, exclude list is [], retry=0 2024-11-14T12:54:27,101 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36021,DS-90f4821d-4e9a-4239-9e98-c91b432844d1,DISK] 2024-11-14T12:54:27,101 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36021,DS-90f4821d-4e9a-4239-9e98-c91b432844d1,DISK] 2024-11-14T12:54:27,101 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35109,DS-111c8733-6272-4257-b572-55835d3a523b,DISK] 2024-11-14T12:54:27,101 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35109,DS-111c8733-6272-4257-b572-55835d3a523b,DISK] 2024-11-14T12:54:27,101 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43871,DS-9aae07ee-1142-4a11-b0be-2c50c0112df3,DISK] 2024-11-14T12:54:27,102 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43871,DS-9aae07ee-1142-4a11-b0be-2c50c0112df3,DISK] 2024-11-14T12:54:27,104 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35109,DS-111c8733-6272-4257-b572-55835d3a523b,DISK] 2024-11-14T12:54:27,104 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43871,DS-9aae07ee-1142-4a11-b0be-2c50c0112df3,DISK] 2024-11-14T12:54:27,104 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36021,DS-90f4821d-4e9a-4239-9e98-c91b432844d1,DISK] 2024-11-14T12:54:27,119 INFO [RS:2;669ef94e8b26:44819 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/WALs/669ef94e8b26,44819,1731588865173/669ef94e8b26%2C44819%2C1731588865173.1731588867078 2024-11-14T12:54:27,121 INFO [RS:0;669ef94e8b26:43475 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/WALs/669ef94e8b26,43475,1731588864980/669ef94e8b26%2C43475%2C1731588864980.1731588867077 2024-11-14T12:54:27,122 INFO [RS:1;669ef94e8b26:46877 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/WALs/669ef94e8b26,46877,1731588865107/669ef94e8b26%2C46877%2C1731588865107.1731588867083 2024-11-14T12:54:27,123 DEBUG [RS:2;669ef94e8b26:44819 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38479:38479),(127.0.0.1/127.0.0.1:44851:44851),(127.0.0.1/127.0.0.1:39561:39561)] 2024-11-14T12:54:27,127 DEBUG [RS:1;669ef94e8b26:46877 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44851:44851),(127.0.0.1/127.0.0.1:39561:39561),(127.0.0.1/127.0.0.1:38479:38479)] 2024-11-14T12:54:27,127 DEBUG [RS:0;669ef94e8b26:43475 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44851:44851),(127.0.0.1/127.0.0.1:39561:39561),(127.0.0.1/127.0.0.1:38479:38479)] 2024-11-14T12:54:27,235 DEBUG [669ef94e8b26:45309 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-14T12:54:27,246 DEBUG [669ef94e8b26:45309 {}] balancer.BalancerClusterState(204): Hosts are {669ef94e8b26=0} racks are {/default-rack=0} 2024-11-14T12:54:27,254 DEBUG [669ef94e8b26:45309 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-14T12:54:27,255 DEBUG [669ef94e8b26:45309 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-14T12:54:27,255 DEBUG [669ef94e8b26:45309 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-14T12:54:27,255 DEBUG [669ef94e8b26:45309 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-14T12:54:27,255 DEBUG [669ef94e8b26:45309 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-14T12:54:27,255 DEBUG [669ef94e8b26:45309 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-14T12:54:27,255 INFO [669ef94e8b26:45309 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-14T12:54:27,255 INFO [669ef94e8b26:45309 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-14T12:54:27,255 INFO [669ef94e8b26:45309 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-14T12:54:27,255 DEBUG [669ef94e8b26:45309 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-14T12:54:27,264 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=669ef94e8b26,44819,1731588865173 2024-11-14T12:54:27,271 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 669ef94e8b26,44819,1731588865173, state=OPENING 2024-11-14T12:54:27,297 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-14T12:54:27,305 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46877-0x1013ab2e72b0002, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T12:54:27,305 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44819-0x1013ab2e72b0003, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T12:54:27,305 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45309-0x1013ab2e72b0000, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T12:54:27,306 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43475-0x1013ab2e72b0001, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T12:54:27,307 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:27,307 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:27,307 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:27,307 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:27,309 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T12:54:27,311 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=669ef94e8b26,44819,1731588865173}] 2024-11-14T12:54:27,485 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-14T12:54:27,487 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53989, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-14T12:54:27,499 INFO [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-14T12:54:27,499 INFO [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T12:54:27,500 INFO [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-14T12:54:27,503 INFO [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=669ef94e8b26%2C44819%2C1731588865173.meta, suffix=.meta, logDir=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/WALs/669ef94e8b26,44819,1731588865173, archiveDir=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/oldWALs, maxLogs=32 2024-11-14T12:54:27,519 DEBUG [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/WALs/669ef94e8b26,44819,1731588865173/669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, exclude list is [], retry=0 2024-11-14T12:54:27,523 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36021,DS-90f4821d-4e9a-4239-9e98-c91b432844d1,DISK] 2024-11-14T12:54:27,523 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35109,DS-111c8733-6272-4257-b572-55835d3a523b,DISK] 2024-11-14T12:54:27,523 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43871,DS-9aae07ee-1142-4a11-b0be-2c50c0112df3,DISK] 2024-11-14T12:54:27,527 INFO [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/WALs/669ef94e8b26,44819,1731588865173/669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta 2024-11-14T12:54:27,527 DEBUG [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39561:39561),(127.0.0.1/127.0.0.1:38479:38479),(127.0.0.1/127.0.0.1:44851:44851)] 2024-11-14T12:54:27,527 DEBUG [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-14T12:54:27,529 DEBUG [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-14T12:54:27,531 DEBUG [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-14T12:54:27,536 INFO [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-14T12:54:27,539 DEBUG [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-14T12:54:27,540 DEBUG [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:27,540 DEBUG [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-14T12:54:27,540 DEBUG [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-14T12:54:27,543 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-14T12:54:27,545 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T12:54:27,545 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T12:54:27,546 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T12:54:27,546 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-14T12:54:27,547 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T12:54:27,548 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T12:54:27,549 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T12:54:27,549 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-14T12:54:27,550 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T12:54:27,551 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T12:54:27,551 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T12:54:27,552 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-14T12:54:27,553 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T12:54:27,553 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T12:54:27,554 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T12:54:27,554 DEBUG [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T12:54:27,556 DEBUG [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/hbase/meta/1588230740 2024-11-14T12:54:27,559 DEBUG [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/hbase/meta/1588230740 2024-11-14T12:54:27,566 DEBUG [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T12:54:27,566 DEBUG [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T12:54:27,567 DEBUG [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-14T12:54:27,569 DEBUG [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T12:54:27,571 INFO [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69210077, jitterRate=0.031310513615608215}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-14T12:54:27,571 DEBUG [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-14T12:54:27,572 DEBUG [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731588867541Writing region info on filesystem at 1731588867541Initializing all the Stores at 1731588867543 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731588867543Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731588867543Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731588867543Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731588867543Cleaning up temporary data from old regions at 1731588867566 (+23 ms)Running coprocessor post-open hooks at 1731588867571 (+5 ms)Region opened successfully at 1731588867572 (+1 ms) 2024-11-14T12:54:27,579 INFO [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731588867477 2024-11-14T12:54:27,591 DEBUG [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-14T12:54:27,592 INFO [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-14T12:54:27,593 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=669ef94e8b26,44819,1731588865173 2024-11-14T12:54:27,596 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 669ef94e8b26,44819,1731588865173, state=OPEN 2024-11-14T12:54:27,605 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46877-0x1013ab2e72b0002, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:27,605 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45309-0x1013ab2e72b0000, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:27,605 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44819-0x1013ab2e72b0003, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:27,605 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43475-0x1013ab2e72b0001, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:27,606 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:27,606 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:27,606 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:27,606 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:27,606 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=669ef94e8b26,44819,1731588865173 2024-11-14T12:54:27,614 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-14T12:54:27,614 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=669ef94e8b26,44819,1731588865173 in 295 msec 2024-11-14T12:54:27,625 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-14T12:54:27,625 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 810 msec 2024-11-14T12:54:27,627 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T12:54:27,627 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-14T12:54:27,646 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T12:54:27,647 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=669ef94e8b26,44819,1731588865173, seqNum=-1] 2024-11-14T12:54:27,672 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T12:54:27,675 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53399, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T12:54:27,705 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2370 sec 2024-11-14T12:54:27,705 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731588867705, completionTime=-1 2024-11-14T12:54:27,709 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-14T12:54:27,709 DEBUG [master/669ef94e8b26:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-14T12:54:27,771 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=3 2024-11-14T12:54:27,771 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731588927771 2024-11-14T12:54:27,771 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731588987771 2024-11-14T12:54:27,771 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 62 msec 2024-11-14T12:54:27,773 DEBUG [master/669ef94e8b26:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-14T12:54:27,782 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=669ef94e8b26,45309,1731588864185-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T12:54:27,782 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=669ef94e8b26,45309,1731588864185-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T12:54:27,782 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=669ef94e8b26,45309,1731588864185-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T12:54:27,784 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-669ef94e8b26:45309, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T12:54:27,785 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-14T12:54:27,795 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-14T12:54:27,802 DEBUG [master/669ef94e8b26:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-14T12:54:27,825 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.528sec 2024-11-14T12:54:27,827 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-14T12:54:27,828 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-14T12:54:27,830 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-14T12:54:27,830 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-14T12:54:27,831 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-14T12:54:27,832 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=669ef94e8b26,45309,1731588864185-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T12:54:27,832 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=669ef94e8b26,45309,1731588864185-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-14T12:54:27,837 DEBUG [master/669ef94e8b26:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-14T12:54:27,838 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-14T12:54:27,838 INFO [master/669ef94e8b26:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=669ef94e8b26,45309,1731588864185-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T12:54:27,929 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ff0a408, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T12:54:27,932 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-14T12:54:27,932 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-14T12:54:27,936 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 669ef94e8b26,45309,-1 for getting cluster id 2024-11-14T12:54:27,938 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T12:54:27,945 DEBUG [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e7c3c559-5520-4497-bea0-7e8e506d06ae' 2024-11-14T12:54:27,948 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T12:54:27,948 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e7c3c559-5520-4497-bea0-7e8e506d06ae" 2024-11-14T12:54:27,949 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52b0024a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T12:54:27,949 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [669ef94e8b26,45309,-1] 2024-11-14T12:54:27,952 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T12:54:27,953 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T12:54:27,955 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47766, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T12:54:27,958 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6238c979, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T12:54:27,959 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T12:54:27,967 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=669ef94e8b26,44819,1731588865173, seqNum=-1] 2024-11-14T12:54:27,968 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T12:54:27,970 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49004, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T12:54:27,995 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=669ef94e8b26,45309,1731588864185 2024-11-14T12:54:28,000 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T12:54:28,008 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 669ef94e8b26,45309,1731588864185 2024-11-14T12:54:28,011 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@68abe514 2024-11-14T12:54:28,013 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T12:54:28,015 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47770, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T12:54:28,022 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45309 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'test', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T12:54:28,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45309 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test 2024-11-14T12:54:28,037 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T12:54:28,039 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45309 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "test" procId is: 4 2024-11-14T12:54:28,039 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T12:54:28,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45309 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T12:54:28,047 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T12:54:28,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741837_1013 (size=358) 2024-11-14T12:54:28,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36021 is added to blk_1073741837_1013 (size=358) 2024-11-14T12:54:28,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43871 is added to blk_1073741837_1013 (size=358) 2024-11-14T12:54:28,088 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a572ae08c8738aa198c40f32aa4b5f96, NAME => 'test,,1731588868017.a572ae08c8738aa198c40f32aa4b5f96.', STARTKEY => '', ENDKEY => 'bbb'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625 2024-11-14T12:54:28,093 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => c2ccfc8c17ab1571bc4ea6d526017f9f, NAME => 'test,bbb,1731588868017.c2ccfc8c17ab1571bc4ea6d526017f9f.', STARTKEY => 'bbb', ENDKEY => 'ccc'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625 2024-11-14T12:54:28,094 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 0fd0512a8a2f36befd3453e1140df3af, NAME => 'test,ccc,1731588868017.0fd0512a8a2f36befd3453e1140df3af.', STARTKEY => 'ccc', ENDKEY => 'ddd'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625 2024-11-14T12:54:28,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36021 is added to blk_1073741838_1014 (size=42) 2024-11-14T12:54:28,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741838_1014 (size=42) 2024-11-14T12:54:28,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43871 is added to blk_1073741838_1014 (size=42) 2024-11-14T12:54:28,162 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,,1731588868017.a572ae08c8738aa198c40f32aa4b5f96.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:28,162 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing a572ae08c8738aa198c40f32aa4b5f96, disabling compactions & flushes 2024-11-14T12:54:28,162 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,,1731588868017.a572ae08c8738aa198c40f32aa4b5f96. 2024-11-14T12:54:28,162 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1731588868017.a572ae08c8738aa198c40f32aa4b5f96. 2024-11-14T12:54:28,163 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,,1731588868017.a572ae08c8738aa198c40f32aa4b5f96. after waiting 0 ms 2024-11-14T12:54:28,163 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,,1731588868017.a572ae08c8738aa198c40f32aa4b5f96. 2024-11-14T12:54:28,163 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,,1731588868017.a572ae08c8738aa198c40f32aa4b5f96. 2024-11-14T12:54:28,163 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for a572ae08c8738aa198c40f32aa4b5f96: Waiting for close lock at 1731588868162Disabling compacts and flushes for region at 1731588868162Disabling writes for close at 1731588868163 (+1 ms)Writing region close event to WAL at 1731588868163Closed at 1731588868163 2024-11-14T12:54:28,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45309 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T12:54:28,164 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b8044551b940cf51bfa7c7633c4df21d, NAME => 'test,ddd,1731588868017.b8044551b940cf51bfa7c7633c4df21d.', STARTKEY => 'ddd', ENDKEY => 'eee'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625 2024-11-14T12:54:28,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36021 is added to blk_1073741839_1015 (size=45) 2024-11-14T12:54:28,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43871 is added to blk_1073741839_1015 (size=45) 2024-11-14T12:54:28,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741839_1015 (size=45) 2024-11-14T12:54:28,195 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,bbb,1731588868017.c2ccfc8c17ab1571bc4ea6d526017f9f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:28,195 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing c2ccfc8c17ab1571bc4ea6d526017f9f, disabling compactions & flushes 2024-11-14T12:54:28,195 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,bbb,1731588868017.c2ccfc8c17ab1571bc4ea6d526017f9f. 2024-11-14T12:54:28,195 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1731588868017.c2ccfc8c17ab1571bc4ea6d526017f9f. 2024-11-14T12:54:28,195 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1731588868017.c2ccfc8c17ab1571bc4ea6d526017f9f. after waiting 0 ms 2024-11-14T12:54:28,195 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1731588868017.c2ccfc8c17ab1571bc4ea6d526017f9f. 2024-11-14T12:54:28,195 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,bbb,1731588868017.c2ccfc8c17ab1571bc4ea6d526017f9f. 2024-11-14T12:54:28,196 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for c2ccfc8c17ab1571bc4ea6d526017f9f: Waiting for close lock at 1731588868195Disabling compacts and flushes for region at 1731588868195Disabling writes for close at 1731588868195Writing region close event to WAL at 1731588868195Closed at 1731588868195 2024-11-14T12:54:28,196 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 23f7fbc112f1353900c1181e3ed36707, NAME => 'test,eee,1731588868017.23f7fbc112f1353900c1181e3ed36707.', STARTKEY => 'eee', ENDKEY => 'fff'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625 2024-11-14T12:54:28,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43871 is added to blk_1073741840_1016 (size=45) 2024-11-14T12:54:28,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36021 is added to blk_1073741840_1016 (size=45) 2024-11-14T12:54:28,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741840_1016 (size=45) 2024-11-14T12:54:28,210 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ccc,1731588868017.0fd0512a8a2f36befd3453e1140df3af.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:28,210 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 0fd0512a8a2f36befd3453e1140df3af, disabling compactions & flushes 2024-11-14T12:54:28,210 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ccc,1731588868017.0fd0512a8a2f36befd3453e1140df3af. 2024-11-14T12:54:28,210 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1731588868017.0fd0512a8a2f36befd3453e1140df3af. 2024-11-14T12:54:28,211 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1731588868017.0fd0512a8a2f36befd3453e1140df3af. after waiting 0 ms 2024-11-14T12:54:28,211 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1731588868017.0fd0512a8a2f36befd3453e1140df3af. 2024-11-14T12:54:28,211 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ccc,1731588868017.0fd0512a8a2f36befd3453e1140df3af. 2024-11-14T12:54:28,211 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 0fd0512a8a2f36befd3453e1140df3af: Waiting for close lock at 1731588868210Disabling compacts and flushes for region at 1731588868210Disabling writes for close at 1731588868211 (+1 ms)Writing region close event to WAL at 1731588868211Closed at 1731588868211 2024-11-14T12:54:28,212 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => bf47be17fd22bc204db28da07b30af2c, NAME => 'test,fff,1731588868017.bf47be17fd22bc204db28da07b30af2c.', STARTKEY => 'fff', ENDKEY => 'ggg'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625 2024-11-14T12:54:28,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36021 is added to blk_1073741841_1017 (size=45) 2024-11-14T12:54:28,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741841_1017 (size=45) 2024-11-14T12:54:28,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43871 is added to blk_1073741841_1017 (size=45) 2024-11-14T12:54:28,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741842_1018 (size=45) 2024-11-14T12:54:28,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36021 is added to blk_1073741842_1018 (size=45) 2024-11-14T12:54:28,266 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,eee,1731588868017.23f7fbc112f1353900c1181e3ed36707.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:28,267 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 23f7fbc112f1353900c1181e3ed36707, disabling compactions & flushes 2024-11-14T12:54:28,267 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,eee,1731588868017.23f7fbc112f1353900c1181e3ed36707. 2024-11-14T12:54:28,267 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1731588868017.23f7fbc112f1353900c1181e3ed36707. 2024-11-14T12:54:28,267 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,eee,1731588868017.23f7fbc112f1353900c1181e3ed36707. after waiting 0 ms 2024-11-14T12:54:28,267 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,eee,1731588868017.23f7fbc112f1353900c1181e3ed36707. 2024-11-14T12:54:28,267 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,eee,1731588868017.23f7fbc112f1353900c1181e3ed36707. 2024-11-14T12:54:28,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43871 is added to blk_1073741842_1018 (size=45) 2024-11-14T12:54:28,267 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 23f7fbc112f1353900c1181e3ed36707: Waiting for close lock at 1731588868266Disabling compacts and flushes for region at 1731588868266Disabling writes for close at 1731588868267 (+1 ms)Writing region close event to WAL at 1731588868267Closed at 1731588868267 2024-11-14T12:54:28,268 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => f76ec25f51f69b1106342a7dfae65fb6, NAME => 'test,ggg,1731588868017.f76ec25f51f69b1106342a7dfae65fb6.', STARTKEY => 'ggg', ENDKEY => 'hhh'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625 2024-11-14T12:54:28,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36021 is added to blk_1073741843_1019 (size=45) 2024-11-14T12:54:28,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43871 is added to blk_1073741843_1019 (size=45) 2024-11-14T12:54:28,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741843_1019 (size=45) 2024-11-14T12:54:28,283 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,fff,1731588868017.bf47be17fd22bc204db28da07b30af2c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:28,283 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing bf47be17fd22bc204db28da07b30af2c, disabling compactions & flushes 2024-11-14T12:54:28,283 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,fff,1731588868017.bf47be17fd22bc204db28da07b30af2c. 2024-11-14T12:54:28,283 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1731588868017.bf47be17fd22bc204db28da07b30af2c. 2024-11-14T12:54:28,283 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,fff,1731588868017.bf47be17fd22bc204db28da07b30af2c. after waiting 0 ms 2024-11-14T12:54:28,283 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,fff,1731588868017.bf47be17fd22bc204db28da07b30af2c. 2024-11-14T12:54:28,283 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,fff,1731588868017.bf47be17fd22bc204db28da07b30af2c. 2024-11-14T12:54:28,283 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for bf47be17fd22bc204db28da07b30af2c: Waiting for close lock at 1731588868283Disabling compacts and flushes for region at 1731588868283Disabling writes for close at 1731588868283Writing region close event to WAL at 1731588868283Closed at 1731588868283 2024-11-14T12:54:28,284 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => cdc4b47ed9fe07b56101f9e0e17abcd7, NAME => 'test,hhh,1731588868017.cdc4b47ed9fe07b56101f9e0e17abcd7.', STARTKEY => 'hhh', ENDKEY => 'iii'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625 2024-11-14T12:54:28,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741844_1020 (size=45) 2024-11-14T12:54:28,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36021 is added to blk_1073741844_1020 (size=45) 2024-11-14T12:54:28,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43871 is added to blk_1073741844_1020 (size=45) 2024-11-14T12:54:28,319 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ggg,1731588868017.f76ec25f51f69b1106342a7dfae65fb6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:28,319 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing f76ec25f51f69b1106342a7dfae65fb6, disabling compactions & flushes 2024-11-14T12:54:28,319 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ggg,1731588868017.f76ec25f51f69b1106342a7dfae65fb6. 2024-11-14T12:54:28,320 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1731588868017.f76ec25f51f69b1106342a7dfae65fb6. 2024-11-14T12:54:28,320 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1731588868017.f76ec25f51f69b1106342a7dfae65fb6. after waiting 0 ms 2024-11-14T12:54:28,320 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1731588868017.f76ec25f51f69b1106342a7dfae65fb6. 2024-11-14T12:54:28,320 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ggg,1731588868017.f76ec25f51f69b1106342a7dfae65fb6. 2024-11-14T12:54:28,320 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for f76ec25f51f69b1106342a7dfae65fb6: Waiting for close lock at 1731588868319Disabling compacts and flushes for region at 1731588868319Disabling writes for close at 1731588868320 (+1 ms)Writing region close event to WAL at 1731588868320Closed at 1731588868320 2024-11-14T12:54:28,321 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 03f289a2805f0b2f037581e9390ba191, NAME => 'test,iii,1731588868017.03f289a2805f0b2f037581e9390ba191.', STARTKEY => 'iii', ENDKEY => 'jjj'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625 2024-11-14T12:54:28,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43871 is added to blk_1073741845_1021 (size=45) 2024-11-14T12:54:28,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741845_1021 (size=45) 2024-11-14T12:54:28,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36021 is added to blk_1073741845_1021 (size=45) 2024-11-14T12:54:28,349 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,hhh,1731588868017.cdc4b47ed9fe07b56101f9e0e17abcd7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:28,349 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing cdc4b47ed9fe07b56101f9e0e17abcd7, disabling compactions & flushes 2024-11-14T12:54:28,349 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,hhh,1731588868017.cdc4b47ed9fe07b56101f9e0e17abcd7. 2024-11-14T12:54:28,349 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1731588868017.cdc4b47ed9fe07b56101f9e0e17abcd7. 2024-11-14T12:54:28,349 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1731588868017.cdc4b47ed9fe07b56101f9e0e17abcd7. after waiting 0 ms 2024-11-14T12:54:28,350 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1731588868017.cdc4b47ed9fe07b56101f9e0e17abcd7. 2024-11-14T12:54:28,350 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,hhh,1731588868017.cdc4b47ed9fe07b56101f9e0e17abcd7. 2024-11-14T12:54:28,350 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for cdc4b47ed9fe07b56101f9e0e17abcd7: Waiting for close lock at 1731588868349Disabling compacts and flushes for region at 1731588868349Disabling writes for close at 1731588868350 (+1 ms)Writing region close event to WAL at 1731588868350Closed at 1731588868350 2024-11-14T12:54:28,351 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 1358fa84af33d1799bd11ba7f4959a37, NAME => 'test,jjj,1731588868017.1358fa84af33d1799bd11ba7f4959a37.', STARTKEY => 'jjj', ENDKEY => 'kkk'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625 2024-11-14T12:54:28,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45309 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T12:54:28,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43871 is added to blk_1073741846_1022 (size=45) 2024-11-14T12:54:28,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741846_1022 (size=45) 2024-11-14T12:54:28,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36021 is added to blk_1073741846_1022 (size=45) 2024-11-14T12:54:28,386 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,iii,1731588868017.03f289a2805f0b2f037581e9390ba191.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:28,386 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 03f289a2805f0b2f037581e9390ba191, disabling compactions & flushes 2024-11-14T12:54:28,386 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,iii,1731588868017.03f289a2805f0b2f037581e9390ba191. 2024-11-14T12:54:28,386 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1731588868017.03f289a2805f0b2f037581e9390ba191. 2024-11-14T12:54:28,386 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,iii,1731588868017.03f289a2805f0b2f037581e9390ba191. after waiting 0 ms 2024-11-14T12:54:28,386 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,iii,1731588868017.03f289a2805f0b2f037581e9390ba191. 2024-11-14T12:54:28,386 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,iii,1731588868017.03f289a2805f0b2f037581e9390ba191. 2024-11-14T12:54:28,387 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 03f289a2805f0b2f037581e9390ba191: Waiting for close lock at 1731588868386Disabling compacts and flushes for region at 1731588868386Disabling writes for close at 1731588868386Writing region close event to WAL at 1731588868386Closed at 1731588868386 2024-11-14T12:54:28,387 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 333a3eba2bb2d318c041beb4642f5ef3, NAME => 'test,kkk,1731588868017.333a3eba2bb2d318c041beb4642f5ef3.', STARTKEY => 'kkk', ENDKEY => 'lll'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625 2024-11-14T12:54:28,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43871 is added to blk_1073741847_1023 (size=45) 2024-11-14T12:54:28,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741847_1023 (size=45) 2024-11-14T12:54:28,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36021 is added to blk_1073741847_1023 (size=45) 2024-11-14T12:54:28,398 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,jjj,1731588868017.1358fa84af33d1799bd11ba7f4959a37.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:28,401 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 1358fa84af33d1799bd11ba7f4959a37, disabling compactions & flushes 2024-11-14T12:54:28,402 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,jjj,1731588868017.1358fa84af33d1799bd11ba7f4959a37. 2024-11-14T12:54:28,402 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1731588868017.1358fa84af33d1799bd11ba7f4959a37. 2024-11-14T12:54:28,402 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1731588868017.1358fa84af33d1799bd11ba7f4959a37. after waiting 0 ms 2024-11-14T12:54:28,402 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1731588868017.1358fa84af33d1799bd11ba7f4959a37. 2024-11-14T12:54:28,402 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,jjj,1731588868017.1358fa84af33d1799bd11ba7f4959a37. 2024-11-14T12:54:28,402 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 1358fa84af33d1799bd11ba7f4959a37: Waiting for close lock at 1731588868401Disabling compacts and flushes for region at 1731588868401Disabling writes for close at 1731588868402 (+1 ms)Writing region close event to WAL at 1731588868402Closed at 1731588868402 2024-11-14T12:54:28,403 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 64b5bd4e6112417a55e871b36551c5b8, NAME => 'test,lll,1731588868017.64b5bd4e6112417a55e871b36551c5b8.', STARTKEY => 'lll', ENDKEY => 'mmm'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625 2024-11-14T12:54:28,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741848_1024 (size=45) 2024-11-14T12:54:28,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43871 is added to blk_1073741848_1024 (size=45) 2024-11-14T12:54:28,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36021 is added to blk_1073741848_1024 (size=45) 2024-11-14T12:54:28,420 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,kkk,1731588868017.333a3eba2bb2d318c041beb4642f5ef3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:28,421 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 333a3eba2bb2d318c041beb4642f5ef3, disabling compactions & flushes 2024-11-14T12:54:28,421 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,kkk,1731588868017.333a3eba2bb2d318c041beb4642f5ef3. 2024-11-14T12:54:28,421 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1731588868017.333a3eba2bb2d318c041beb4642f5ef3. 2024-11-14T12:54:28,421 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1731588868017.333a3eba2bb2d318c041beb4642f5ef3. after waiting 0 ms 2024-11-14T12:54:28,421 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1731588868017.333a3eba2bb2d318c041beb4642f5ef3. 2024-11-14T12:54:28,421 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,kkk,1731588868017.333a3eba2bb2d318c041beb4642f5ef3. 2024-11-14T12:54:28,421 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 333a3eba2bb2d318c041beb4642f5ef3: Waiting for close lock at 1731588868421Disabling compacts and flushes for region at 1731588868421Disabling writes for close at 1731588868421Writing region close event to WAL at 1731588868421Closed at 1731588868421 2024-11-14T12:54:28,422 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => e90cc230c4f13fece446e3e8420dddbe, NAME => 'test,mmm,1731588868017.e90cc230c4f13fece446e3e8420dddbe.', STARTKEY => 'mmm', ENDKEY => 'nnn'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625 2024-11-14T12:54:28,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43871 is added to blk_1073741849_1025 (size=45) 2024-11-14T12:54:28,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741849_1025 (size=45) 2024-11-14T12:54:28,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36021 is added to blk_1073741849_1025 (size=45) 2024-11-14T12:54:28,436 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,lll,1731588868017.64b5bd4e6112417a55e871b36551c5b8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:28,436 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 64b5bd4e6112417a55e871b36551c5b8, disabling compactions & flushes 2024-11-14T12:54:28,436 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,lll,1731588868017.64b5bd4e6112417a55e871b36551c5b8. 2024-11-14T12:54:28,436 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1731588868017.64b5bd4e6112417a55e871b36551c5b8. 2024-11-14T12:54:28,436 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,lll,1731588868017.64b5bd4e6112417a55e871b36551c5b8. after waiting 0 ms 2024-11-14T12:54:28,436 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,lll,1731588868017.64b5bd4e6112417a55e871b36551c5b8. 2024-11-14T12:54:28,436 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,lll,1731588868017.64b5bd4e6112417a55e871b36551c5b8. 2024-11-14T12:54:28,437 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 64b5bd4e6112417a55e871b36551c5b8: Waiting for close lock at 1731588868436Disabling compacts and flushes for region at 1731588868436Disabling writes for close at 1731588868436Writing region close event to WAL at 1731588868436Closed at 1731588868436 2024-11-14T12:54:28,438 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 4f16209fa220cf1b7a18b891e4fe53c6, NAME => 'test,nnn,1731588868017.4f16209fa220cf1b7a18b891e4fe53c6.', STARTKEY => 'nnn', ENDKEY => 'ooo'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625 2024-11-14T12:54:28,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741850_1026 (size=45) 2024-11-14T12:54:28,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43871 is added to blk_1073741850_1026 (size=45) 2024-11-14T12:54:28,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36021 is added to blk_1073741850_1026 (size=45) 2024-11-14T12:54:28,458 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,mmm,1731588868017.e90cc230c4f13fece446e3e8420dddbe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:28,458 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing e90cc230c4f13fece446e3e8420dddbe, disabling compactions & flushes 2024-11-14T12:54:28,458 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,mmm,1731588868017.e90cc230c4f13fece446e3e8420dddbe. 2024-11-14T12:54:28,458 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1731588868017.e90cc230c4f13fece446e3e8420dddbe. 2024-11-14T12:54:28,458 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1731588868017.e90cc230c4f13fece446e3e8420dddbe. after waiting 0 ms 2024-11-14T12:54:28,459 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1731588868017.e90cc230c4f13fece446e3e8420dddbe. 2024-11-14T12:54:28,459 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,mmm,1731588868017.e90cc230c4f13fece446e3e8420dddbe. 2024-11-14T12:54:28,459 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for e90cc230c4f13fece446e3e8420dddbe: Waiting for close lock at 1731588868458Disabling compacts and flushes for region at 1731588868458Disabling writes for close at 1731588868458Writing region close event to WAL at 1731588868459 (+1 ms)Closed at 1731588868459 2024-11-14T12:54:28,459 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 338b47e35e9a8b9dd3045d365aaf47bd, NAME => 'test,ooo,1731588868017.338b47e35e9a8b9dd3045d365aaf47bd.', STARTKEY => 'ooo', ENDKEY => 'ppp'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625 2024-11-14T12:54:28,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36021 is added to blk_1073741851_1027 (size=45) 2024-11-14T12:54:28,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741851_1027 (size=45) 2024-11-14T12:54:28,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43871 is added to blk_1073741851_1027 (size=45) 2024-11-14T12:54:28,473 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,nnn,1731588868017.4f16209fa220cf1b7a18b891e4fe53c6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:28,473 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 4f16209fa220cf1b7a18b891e4fe53c6, disabling compactions & flushes 2024-11-14T12:54:28,473 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,nnn,1731588868017.4f16209fa220cf1b7a18b891e4fe53c6. 2024-11-14T12:54:28,473 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1731588868017.4f16209fa220cf1b7a18b891e4fe53c6. 2024-11-14T12:54:28,473 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1731588868017.4f16209fa220cf1b7a18b891e4fe53c6. after waiting 0 ms 2024-11-14T12:54:28,473 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1731588868017.4f16209fa220cf1b7a18b891e4fe53c6. 2024-11-14T12:54:28,473 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,nnn,1731588868017.4f16209fa220cf1b7a18b891e4fe53c6. 2024-11-14T12:54:28,473 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 4f16209fa220cf1b7a18b891e4fe53c6: Waiting for close lock at 1731588868473Disabling compacts and flushes for region at 1731588868473Disabling writes for close at 1731588868473Writing region close event to WAL at 1731588868473Closed at 1731588868473 2024-11-14T12:54:28,474 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => ea6e95809b33f853859fa227179bfe85, NAME => 'test,ppp,1731588868017.ea6e95809b33f853859fa227179bfe85.', STARTKEY => 'ppp', ENDKEY => 'qqq'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625 2024-11-14T12:54:28,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741852_1028 (size=45) 2024-11-14T12:54:28,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36021 is added to blk_1073741852_1028 (size=45) 2024-11-14T12:54:28,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43871 is added to blk_1073741852_1028 (size=45) 2024-11-14T12:54:28,484 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ooo,1731588868017.338b47e35e9a8b9dd3045d365aaf47bd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:28,485 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 338b47e35e9a8b9dd3045d365aaf47bd, disabling compactions & flushes 2024-11-14T12:54:28,485 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ooo,1731588868017.338b47e35e9a8b9dd3045d365aaf47bd. 2024-11-14T12:54:28,485 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1731588868017.338b47e35e9a8b9dd3045d365aaf47bd. 2024-11-14T12:54:28,485 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1731588868017.338b47e35e9a8b9dd3045d365aaf47bd. after waiting 0 ms 2024-11-14T12:54:28,485 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1731588868017.338b47e35e9a8b9dd3045d365aaf47bd. 2024-11-14T12:54:28,485 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ooo,1731588868017.338b47e35e9a8b9dd3045d365aaf47bd. 2024-11-14T12:54:28,486 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 338b47e35e9a8b9dd3045d365aaf47bd: Waiting for close lock at 1731588868485Disabling compacts and flushes for region at 1731588868485Disabling writes for close at 1731588868485Writing region close event to WAL at 1731588868485Closed at 1731588868485 2024-11-14T12:54:28,487 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 18ddd925c95b1b23296976da04f97e83, NAME => 'test,qqq,1731588868017.18ddd925c95b1b23296976da04f97e83.', STARTKEY => 'qqq', ENDKEY => 'rrr'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625 2024-11-14T12:54:28,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43871 is added to blk_1073741853_1029 (size=45) 2024-11-14T12:54:28,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741853_1029 (size=45) 2024-11-14T12:54:28,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36021 is added to blk_1073741853_1029 (size=45) 2024-11-14T12:54:28,516 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ppp,1731588868017.ea6e95809b33f853859fa227179bfe85.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:28,516 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing ea6e95809b33f853859fa227179bfe85, disabling compactions & flushes 2024-11-14T12:54:28,516 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ppp,1731588868017.ea6e95809b33f853859fa227179bfe85. 2024-11-14T12:54:28,516 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1731588868017.ea6e95809b33f853859fa227179bfe85. 2024-11-14T12:54:28,516 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1731588868017.ea6e95809b33f853859fa227179bfe85. after waiting 0 ms 2024-11-14T12:54:28,516 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1731588868017.ea6e95809b33f853859fa227179bfe85. 2024-11-14T12:54:28,516 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ppp,1731588868017.ea6e95809b33f853859fa227179bfe85. 2024-11-14T12:54:28,516 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for ea6e95809b33f853859fa227179bfe85: Waiting for close lock at 1731588868516Disabling compacts and flushes for region at 1731588868516Disabling writes for close at 1731588868516Writing region close event to WAL at 1731588868516Closed at 1731588868516 2024-11-14T12:54:28,518 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => a5d884ba0737bc572475e022a6ed1bc8, NAME => 'test,rrr,1731588868017.a5d884ba0737bc572475e022a6ed1bc8.', STARTKEY => 'rrr', ENDKEY => 'sss'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625 2024-11-14T12:54:28,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741854_1030 (size=45) 2024-11-14T12:54:28,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43871 is added to blk_1073741854_1030 (size=45) 2024-11-14T12:54:28,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36021 is added to blk_1073741854_1030 (size=45) 2024-11-14T12:54:28,547 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,qqq,1731588868017.18ddd925c95b1b23296976da04f97e83.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:28,547 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 18ddd925c95b1b23296976da04f97e83, disabling compactions & flushes 2024-11-14T12:54:28,547 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,qqq,1731588868017.18ddd925c95b1b23296976da04f97e83. 2024-11-14T12:54:28,547 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1731588868017.18ddd925c95b1b23296976da04f97e83. 2024-11-14T12:54:28,547 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1731588868017.18ddd925c95b1b23296976da04f97e83. after waiting 0 ms 2024-11-14T12:54:28,547 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1731588868017.18ddd925c95b1b23296976da04f97e83. 2024-11-14T12:54:28,548 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,qqq,1731588868017.18ddd925c95b1b23296976da04f97e83. 2024-11-14T12:54:28,548 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 18ddd925c95b1b23296976da04f97e83: Waiting for close lock at 1731588868547Disabling compacts and flushes for region at 1731588868547Disabling writes for close at 1731588868547Writing region close event to WAL at 1731588868547Closed at 1731588868547 2024-11-14T12:54:28,548 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1d33b9e9c25fcfc654d257646677381c, NAME => 'test,sss,1731588868017.1d33b9e9c25fcfc654d257646677381c.', STARTKEY => 'sss', ENDKEY => 'ttt'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625 2024-11-14T12:54:28,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741855_1031 (size=45) 2024-11-14T12:54:28,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43871 is added to blk_1073741855_1031 (size=45) 2024-11-14T12:54:28,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36021 is added to blk_1073741855_1031 (size=45) 2024-11-14T12:54:28,574 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,rrr,1731588868017.a5d884ba0737bc572475e022a6ed1bc8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:28,574 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing a5d884ba0737bc572475e022a6ed1bc8, disabling compactions & flushes 2024-11-14T12:54:28,574 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,rrr,1731588868017.a5d884ba0737bc572475e022a6ed1bc8. 2024-11-14T12:54:28,574 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1731588868017.a5d884ba0737bc572475e022a6ed1bc8. 2024-11-14T12:54:28,574 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1731588868017.a5d884ba0737bc572475e022a6ed1bc8. after waiting 0 ms 2024-11-14T12:54:28,574 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1731588868017.a5d884ba0737bc572475e022a6ed1bc8. 2024-11-14T12:54:28,574 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,rrr,1731588868017.a5d884ba0737bc572475e022a6ed1bc8. 2024-11-14T12:54:28,574 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for a5d884ba0737bc572475e022a6ed1bc8: Waiting for close lock at 1731588868574Disabling compacts and flushes for region at 1731588868574Disabling writes for close at 1731588868574Writing region close event to WAL at 1731588868574Closed at 1731588868574 2024-11-14T12:54:28,575 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => c8ddc4f331afb3a3007ef5c49cdf6ab8, NAME => 'test,ttt,1731588868017.c8ddc4f331afb3a3007ef5c49cdf6ab8.', STARTKEY => 'ttt', ENDKEY => 'uuu'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625 2024-11-14T12:54:28,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43871 is added to blk_1073741856_1032 (size=45) 2024-11-14T12:54:28,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741856_1032 (size=45) 2024-11-14T12:54:28,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36021 is added to blk_1073741856_1032 (size=45) 2024-11-14T12:54:28,601 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,sss,1731588868017.1d33b9e9c25fcfc654d257646677381c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:28,601 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 1d33b9e9c25fcfc654d257646677381c, disabling compactions & flushes 2024-11-14T12:54:28,601 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,sss,1731588868017.1d33b9e9c25fcfc654d257646677381c. 2024-11-14T12:54:28,601 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1731588868017.1d33b9e9c25fcfc654d257646677381c. 2024-11-14T12:54:28,601 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,sss,1731588868017.1d33b9e9c25fcfc654d257646677381c. after waiting 0 ms 2024-11-14T12:54:28,601 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,sss,1731588868017.1d33b9e9c25fcfc654d257646677381c. 2024-11-14T12:54:28,601 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,sss,1731588868017.1d33b9e9c25fcfc654d257646677381c. 2024-11-14T12:54:28,602 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 1d33b9e9c25fcfc654d257646677381c: Waiting for close lock at 1731588868601Disabling compacts and flushes for region at 1731588868601Disabling writes for close at 1731588868601Writing region close event to WAL at 1731588868601Closed at 1731588868601 2024-11-14T12:54:28,602 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => f8c0d62856832eec9f9ca20bb6db602c, NAME => 'test,uuu,1731588868017.f8c0d62856832eec9f9ca20bb6db602c.', STARTKEY => 'uuu', ENDKEY => 'vvv'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625 2024-11-14T12:54:28,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36021 is added to blk_1073741857_1033 (size=45) 2024-11-14T12:54:28,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43871 is added to blk_1073741857_1033 (size=45) 2024-11-14T12:54:28,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741857_1033 (size=45) 2024-11-14T12:54:28,621 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ttt,1731588868017.c8ddc4f331afb3a3007ef5c49cdf6ab8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:28,622 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing c8ddc4f331afb3a3007ef5c49cdf6ab8, disabling compactions & flushes 2024-11-14T12:54:28,622 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ttt,1731588868017.c8ddc4f331afb3a3007ef5c49cdf6ab8. 2024-11-14T12:54:28,622 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1731588868017.c8ddc4f331afb3a3007ef5c49cdf6ab8. 2024-11-14T12:54:28,622 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1731588868017.c8ddc4f331afb3a3007ef5c49cdf6ab8. after waiting 0 ms 2024-11-14T12:54:28,622 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1731588868017.c8ddc4f331afb3a3007ef5c49cdf6ab8. 2024-11-14T12:54:28,622 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ttt,1731588868017.c8ddc4f331afb3a3007ef5c49cdf6ab8. 2024-11-14T12:54:28,622 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for c8ddc4f331afb3a3007ef5c49cdf6ab8: Waiting for close lock at 1731588868621Disabling compacts and flushes for region at 1731588868621Disabling writes for close at 1731588868622 (+1 ms)Writing region close event to WAL at 1731588868622Closed at 1731588868622 2024-11-14T12:54:28,623 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 87eb074dbd46b3bc6e27fb0bf3291d5b, NAME => 'test,vvv,1731588868017.87eb074dbd46b3bc6e27fb0bf3291d5b.', STARTKEY => 'vvv', ENDKEY => 'www'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625 2024-11-14T12:54:28,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741858_1034 (size=45) 2024-11-14T12:54:28,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36021 is added to blk_1073741858_1034 (size=45) 2024-11-14T12:54:28,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43871 is added to blk_1073741858_1034 (size=45) 2024-11-14T12:54:28,633 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,uuu,1731588868017.f8c0d62856832eec9f9ca20bb6db602c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:28,633 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing f8c0d62856832eec9f9ca20bb6db602c, disabling compactions & flushes 2024-11-14T12:54:28,633 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,uuu,1731588868017.f8c0d62856832eec9f9ca20bb6db602c. 2024-11-14T12:54:28,633 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1731588868017.f8c0d62856832eec9f9ca20bb6db602c. 2024-11-14T12:54:28,633 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1731588868017.f8c0d62856832eec9f9ca20bb6db602c. after waiting 0 ms 2024-11-14T12:54:28,633 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1731588868017.f8c0d62856832eec9f9ca20bb6db602c. 2024-11-14T12:54:28,633 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,uuu,1731588868017.f8c0d62856832eec9f9ca20bb6db602c. 2024-11-14T12:54:28,633 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for f8c0d62856832eec9f9ca20bb6db602c: Waiting for close lock at 1731588868633Disabling compacts and flushes for region at 1731588868633Disabling writes for close at 1731588868633Writing region close event to WAL at 1731588868633Closed at 1731588868633 2024-11-14T12:54:28,634 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 22b9f10374fc1ea7a876d476ce0b083d, NAME => 'test,www,1731588868017.22b9f10374fc1ea7a876d476ce0b083d.', STARTKEY => 'www', ENDKEY => 'xxx'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625 2024-11-14T12:54:28,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741859_1035 (size=45) 2024-11-14T12:54:28,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36021 is added to blk_1073741859_1035 (size=45) 2024-11-14T12:54:28,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43871 is added to blk_1073741859_1035 (size=45) 2024-11-14T12:54:28,640 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,vvv,1731588868017.87eb074dbd46b3bc6e27fb0bf3291d5b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:28,640 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 87eb074dbd46b3bc6e27fb0bf3291d5b, disabling compactions & flushes 2024-11-14T12:54:28,640 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,vvv,1731588868017.87eb074dbd46b3bc6e27fb0bf3291d5b. 2024-11-14T12:54:28,640 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1731588868017.87eb074dbd46b3bc6e27fb0bf3291d5b. 2024-11-14T12:54:28,640 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1731588868017.87eb074dbd46b3bc6e27fb0bf3291d5b. after waiting 0 ms 2024-11-14T12:54:28,640 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1731588868017.87eb074dbd46b3bc6e27fb0bf3291d5b. 2024-11-14T12:54:28,640 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,vvv,1731588868017.87eb074dbd46b3bc6e27fb0bf3291d5b. 2024-11-14T12:54:28,640 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 87eb074dbd46b3bc6e27fb0bf3291d5b: Waiting for close lock at 1731588868640Disabling compacts and flushes for region at 1731588868640Disabling writes for close at 1731588868640Writing region close event to WAL at 1731588868640Closed at 1731588868640 2024-11-14T12:54:28,641 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => cadcf21958eaec53ee489ca0918e3ab7, NAME => 'test,xxx,1731588868017.cadcf21958eaec53ee489ca0918e3ab7.', STARTKEY => 'xxx', ENDKEY => 'yyy'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625 2024-11-14T12:54:28,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36021 is added to blk_1073741860_1036 (size=45) 2024-11-14T12:54:28,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741860_1036 (size=45) 2024-11-14T12:54:28,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43871 is added to blk_1073741860_1036 (size=45) 2024-11-14T12:54:28,648 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,www,1731588868017.22b9f10374fc1ea7a876d476ce0b083d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:28,648 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 22b9f10374fc1ea7a876d476ce0b083d, disabling compactions & flushes 2024-11-14T12:54:28,648 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,www,1731588868017.22b9f10374fc1ea7a876d476ce0b083d. 2024-11-14T12:54:28,648 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1731588868017.22b9f10374fc1ea7a876d476ce0b083d. 2024-11-14T12:54:28,648 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,www,1731588868017.22b9f10374fc1ea7a876d476ce0b083d. after waiting 0 ms 2024-11-14T12:54:28,648 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,www,1731588868017.22b9f10374fc1ea7a876d476ce0b083d. 2024-11-14T12:54:28,648 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,www,1731588868017.22b9f10374fc1ea7a876d476ce0b083d. 2024-11-14T12:54:28,648 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 22b9f10374fc1ea7a876d476ce0b083d: Waiting for close lock at 1731588868648Disabling compacts and flushes for region at 1731588868648Disabling writes for close at 1731588868648Writing region close event to WAL at 1731588868648Closed at 1731588868648 2024-11-14T12:54:28,649 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 66b34b9be40614bd68d0cea995e44193, NAME => 'test,yyy,1731588868017.66b34b9be40614bd68d0cea995e44193.', STARTKEY => 'yyy', ENDKEY => 'zzz'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625 2024-11-14T12:54:28,650 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ddd,1731588868017.b8044551b940cf51bfa7c7633c4df21d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:28,650 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing b8044551b940cf51bfa7c7633c4df21d, disabling compactions & flushes 2024-11-14T12:54:28,650 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ddd,1731588868017.b8044551b940cf51bfa7c7633c4df21d. 2024-11-14T12:54:28,650 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1731588868017.b8044551b940cf51bfa7c7633c4df21d. 2024-11-14T12:54:28,650 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1731588868017.b8044551b940cf51bfa7c7633c4df21d. after waiting 0 ms 2024-11-14T12:54:28,650 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1731588868017.b8044551b940cf51bfa7c7633c4df21d. 2024-11-14T12:54:28,651 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ddd,1731588868017.b8044551b940cf51bfa7c7633c4df21d. 2024-11-14T12:54:28,651 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for b8044551b940cf51bfa7c7633c4df21d: Waiting for close lock at 1731588868650Disabling compacts and flushes for region at 1731588868650Disabling writes for close at 1731588868650Writing region close event to WAL at 1731588868651 (+1 ms)Closed at 1731588868651 2024-11-14T12:54:28,651 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2201f29440b3aa3259741bbb374e209d, NAME => 'test,zzz,1731588868017.2201f29440b3aa3259741bbb374e209d.', STARTKEY => 'zzz', ENDKEY => ''}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625 2024-11-14T12:54:28,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43871 is added to blk_1073741861_1037 (size=45) 2024-11-14T12:54:28,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741861_1037 (size=45) 2024-11-14T12:54:28,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36021 is added to blk_1073741861_1037 (size=45) 2024-11-14T12:54:28,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43871 is added to blk_1073741863_1039 (size=42) 2024-11-14T12:54:28,671 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,xxx,1731588868017.cadcf21958eaec53ee489ca0918e3ab7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:28,671 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing cadcf21958eaec53ee489ca0918e3ab7, disabling compactions & flushes 2024-11-14T12:54:28,671 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,xxx,1731588868017.cadcf21958eaec53ee489ca0918e3ab7. 2024-11-14T12:54:28,671 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1731588868017.cadcf21958eaec53ee489ca0918e3ab7. 2024-11-14T12:54:28,671 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1731588868017.cadcf21958eaec53ee489ca0918e3ab7. after waiting 0 ms 2024-11-14T12:54:28,671 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1731588868017.cadcf21958eaec53ee489ca0918e3ab7. 2024-11-14T12:54:28,671 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,xxx,1731588868017.cadcf21958eaec53ee489ca0918e3ab7. 2024-11-14T12:54:28,671 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for cadcf21958eaec53ee489ca0918e3ab7: Waiting for close lock at 1731588868671Disabling compacts and flushes for region at 1731588868671Disabling writes for close at 1731588868671Writing region close event to WAL at 1731588868671Closed at 1731588868671 2024-11-14T12:54:28,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741863_1039 (size=42) 2024-11-14T12:54:28,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36021 is added to blk_1073741863_1039 (size=42) 2024-11-14T12:54:28,673 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,zzz,1731588868017.2201f29440b3aa3259741bbb374e209d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:28,674 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 2201f29440b3aa3259741bbb374e209d, disabling compactions & flushes 2024-11-14T12:54:28,674 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,zzz,1731588868017.2201f29440b3aa3259741bbb374e209d. 2024-11-14T12:54:28,674 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1731588868017.2201f29440b3aa3259741bbb374e209d. 2024-11-14T12:54:28,674 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1731588868017.2201f29440b3aa3259741bbb374e209d. after waiting 0 ms 2024-11-14T12:54:28,674 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1731588868017.2201f29440b3aa3259741bbb374e209d. 2024-11-14T12:54:28,674 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,zzz,1731588868017.2201f29440b3aa3259741bbb374e209d. 2024-11-14T12:54:28,674 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2201f29440b3aa3259741bbb374e209d: Waiting for close lock at 1731588868673Disabling compacts and flushes for region at 1731588868673Disabling writes for close at 1731588868674 (+1 ms)Writing region close event to WAL at 1731588868674Closed at 1731588868674 2024-11-14T12:54:28,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36021 is added to blk_1073741862_1038 (size=45) 2024-11-14T12:54:28,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741862_1038 (size=45) 2024-11-14T12:54:28,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43871 is added to blk_1073741862_1038 (size=45) 2024-11-14T12:54:28,681 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,yyy,1731588868017.66b34b9be40614bd68d0cea995e44193.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:28,682 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 66b34b9be40614bd68d0cea995e44193, disabling compactions & flushes 2024-11-14T12:54:28,682 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,yyy,1731588868017.66b34b9be40614bd68d0cea995e44193. 2024-11-14T12:54:28,682 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1731588868017.66b34b9be40614bd68d0cea995e44193. 2024-11-14T12:54:28,682 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1731588868017.66b34b9be40614bd68d0cea995e44193. after waiting 0 ms 2024-11-14T12:54:28,682 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1731588868017.66b34b9be40614bd68d0cea995e44193. 2024-11-14T12:54:28,682 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,yyy,1731588868017.66b34b9be40614bd68d0cea995e44193. 2024-11-14T12:54:28,682 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 66b34b9be40614bd68d0cea995e44193: Waiting for close lock at 1731588868681Disabling compacts and flushes for region at 1731588868681Disabling writes for close at 1731588868682 (+1 ms)Writing region close event to WAL at 1731588868682Closed at 1731588868682 2024-11-14T12:54:28,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45309 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T12:54:28,685 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T12:54:28,693 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,,1731588868017.a572ae08c8738aa198c40f32aa4b5f96.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1731588868685"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731588868685"}]},"ts":"1731588868685"} 2024-11-14T12:54:28,693 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,bbb,1731588868017.c2ccfc8c17ab1571bc4ea6d526017f9f.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731588868685"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731588868685"}]},"ts":"1731588868685"} 2024-11-14T12:54:28,694 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ccc,1731588868017.0fd0512a8a2f36befd3453e1140df3af.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731588868685"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731588868685"}]},"ts":"1731588868685"} 2024-11-14T12:54:28,694 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,eee,1731588868017.23f7fbc112f1353900c1181e3ed36707.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731588868685"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731588868685"}]},"ts":"1731588868685"} 2024-11-14T12:54:28,694 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,fff,1731588868017.bf47be17fd22bc204db28da07b30af2c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731588868685"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731588868685"}]},"ts":"1731588868685"} 2024-11-14T12:54:28,695 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ggg,1731588868017.f76ec25f51f69b1106342a7dfae65fb6.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731588868685"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731588868685"}]},"ts":"1731588868685"} 2024-11-14T12:54:28,695 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,hhh,1731588868017.cdc4b47ed9fe07b56101f9e0e17abcd7.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731588868685"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731588868685"}]},"ts":"1731588868685"} 2024-11-14T12:54:28,695 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,iii,1731588868017.03f289a2805f0b2f037581e9390ba191.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731588868685"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731588868685"}]},"ts":"1731588868685"} 2024-11-14T12:54:28,695 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,jjj,1731588868017.1358fa84af33d1799bd11ba7f4959a37.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731588868685"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731588868685"}]},"ts":"1731588868685"} 2024-11-14T12:54:28,695 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,kkk,1731588868017.333a3eba2bb2d318c041beb4642f5ef3.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731588868685"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731588868685"}]},"ts":"1731588868685"} 2024-11-14T12:54:28,696 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,lll,1731588868017.64b5bd4e6112417a55e871b36551c5b8.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731588868685"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731588868685"}]},"ts":"1731588868685"} 2024-11-14T12:54:28,696 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,mmm,1731588868017.e90cc230c4f13fece446e3e8420dddbe.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731588868685"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731588868685"}]},"ts":"1731588868685"} 2024-11-14T12:54:28,696 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,nnn,1731588868017.4f16209fa220cf1b7a18b891e4fe53c6.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731588868685"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731588868685"}]},"ts":"1731588868685"} 2024-11-14T12:54:28,696 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ooo,1731588868017.338b47e35e9a8b9dd3045d365aaf47bd.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731588868685"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731588868685"}]},"ts":"1731588868685"} 2024-11-14T12:54:28,696 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ppp,1731588868017.ea6e95809b33f853859fa227179bfe85.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731588868685"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731588868685"}]},"ts":"1731588868685"} 2024-11-14T12:54:28,696 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,qqq,1731588868017.18ddd925c95b1b23296976da04f97e83.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731588868685"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731588868685"}]},"ts":"1731588868685"} 2024-11-14T12:54:28,697 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,rrr,1731588868017.a5d884ba0737bc572475e022a6ed1bc8.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731588868685"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731588868685"}]},"ts":"1731588868685"} 2024-11-14T12:54:28,697 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,sss,1731588868017.1d33b9e9c25fcfc654d257646677381c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731588868685"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731588868685"}]},"ts":"1731588868685"} 2024-11-14T12:54:28,697 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ttt,1731588868017.c8ddc4f331afb3a3007ef5c49cdf6ab8.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731588868685"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731588868685"}]},"ts":"1731588868685"} 2024-11-14T12:54:28,697 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,uuu,1731588868017.f8c0d62856832eec9f9ca20bb6db602c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731588868685"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731588868685"}]},"ts":"1731588868685"} 2024-11-14T12:54:28,697 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,vvv,1731588868017.87eb074dbd46b3bc6e27fb0bf3291d5b.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731588868685"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731588868685"}]},"ts":"1731588868685"} 2024-11-14T12:54:28,697 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,www,1731588868017.22b9f10374fc1ea7a876d476ce0b083d.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731588868685"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731588868685"}]},"ts":"1731588868685"} 2024-11-14T12:54:28,698 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ddd,1731588868017.b8044551b940cf51bfa7c7633c4df21d.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731588868685"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731588868685"}]},"ts":"1731588868685"} 2024-11-14T12:54:28,698 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,xxx,1731588868017.cadcf21958eaec53ee489ca0918e3ab7.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731588868685"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731588868685"}]},"ts":"1731588868685"} 2024-11-14T12:54:28,698 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,zzz,1731588868017.2201f29440b3aa3259741bbb374e209d.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1731588868685"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731588868685"}]},"ts":"1731588868685"} 2024-11-14T12:54:28,698 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,yyy,1731588868017.66b34b9be40614bd68d0cea995e44193.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731588868685"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731588868685"}]},"ts":"1731588868685"} 2024-11-14T12:54:28,763 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 26 regions to meta. 2024-11-14T12:54:28,766 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T12:54:28,768 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731588868766"}]},"ts":"1731588868766"} 2024-11-14T12:54:28,780 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLING in hbase:meta 2024-11-14T12:54:28,781 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {669ef94e8b26=0} racks are {/default-rack=0} 2024-11-14T12:54:28,805 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-14T12:54:28,806 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-14T12:54:28,806 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-14T12:54:28,806 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-14T12:54:28,806 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-14T12:54:28,806 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-14T12:54:28,806 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-14T12:54:28,806 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-14T12:54:28,806 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-14T12:54:28,806 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-14T12:54:28,809 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c2ccfc8c17ab1571bc4ea6d526017f9f, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0fd0512a8a2f36befd3453e1140df3af, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b8044551b940cf51bfa7c7633c4df21d, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=23f7fbc112f1353900c1181e3ed36707, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=bf47be17fd22bc204db28da07b30af2c, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f76ec25f51f69b1106342a7dfae65fb6, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=03f289a2805f0b2f037581e9390ba191, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1358fa84af33d1799bd11ba7f4959a37, ASSIGN}, {pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=333a3eba2bb2d318c041beb4642f5ef3, ASSIGN}, {pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=64b5bd4e6112417a55e871b36551c5b8, ASSIGN}, {pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e90cc230c4f13fece446e3e8420dddbe, ASSIGN}, {pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4f16209fa220cf1b7a18b891e4fe53c6, ASSIGN}, {pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=338b47e35e9a8b9dd3045d365aaf47bd, ASSIGN}, {pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ea6e95809b33f853859fa227179bfe85, ASSIGN}, {pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=18ddd925c95b1b23296976da04f97e83, ASSIGN}, {pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a5d884ba0737bc572475e022a6ed1bc8, ASSIGN}, {pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1d33b9e9c25fcfc654d257646677381c, ASSIGN}, {pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN}, {pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f8c0d62856832eec9f9ca20bb6db602c, ASSIGN}, {pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=87eb074dbd46b3bc6e27fb0bf3291d5b, ASSIGN}, {pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=22b9f10374fc1ea7a876d476ce0b083d, ASSIGN}, {pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=cadcf21958eaec53ee489ca0918e3ab7, ASSIGN}, {pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=66b34b9be40614bd68d0cea995e44193, ASSIGN}, {pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2201f29440b3aa3259741bbb374e209d, ASSIGN}] 2024-11-14T12:54:28,821 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2201f29440b3aa3259741bbb374e209d, ASSIGN 2024-11-14T12:54:28,823 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=2201f29440b3aa3259741bbb374e209d, ASSIGN; state=OFFLINE, location=669ef94e8b26,44819,1731588865173; forceNewPlan=false, retain=false 2024-11-14T12:54:28,826 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=66b34b9be40614bd68d0cea995e44193, ASSIGN 2024-11-14T12:54:28,828 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=66b34b9be40614bd68d0cea995e44193, ASSIGN; state=OFFLINE, location=669ef94e8b26,43475,1731588864980; forceNewPlan=false, retain=false 2024-11-14T12:54:28,832 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=cadcf21958eaec53ee489ca0918e3ab7, ASSIGN 2024-11-14T12:54:28,834 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=cadcf21958eaec53ee489ca0918e3ab7, ASSIGN; state=OFFLINE, location=669ef94e8b26,46877,1731588865107; forceNewPlan=false, retain=false 2024-11-14T12:54:28,837 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=22b9f10374fc1ea7a876d476ce0b083d, ASSIGN 2024-11-14T12:54:28,840 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=22b9f10374fc1ea7a876d476ce0b083d, ASSIGN; state=OFFLINE, location=669ef94e8b26,44819,1731588865173; forceNewPlan=false, retain=false 2024-11-14T12:54:28,843 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=87eb074dbd46b3bc6e27fb0bf3291d5b, ASSIGN 2024-11-14T12:54:28,845 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=87eb074dbd46b3bc6e27fb0bf3291d5b, ASSIGN; state=OFFLINE, location=669ef94e8b26,46877,1731588865107; forceNewPlan=false, retain=false 2024-11-14T12:54:28,847 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f8c0d62856832eec9f9ca20bb6db602c, ASSIGN 2024-11-14T12:54:28,849 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f8c0d62856832eec9f9ca20bb6db602c, ASSIGN; state=OFFLINE, location=669ef94e8b26,43475,1731588864980; forceNewPlan=false, retain=false 2024-11-14T12:54:28,852 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN 2024-11-14T12:54:28,855 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN; state=OFFLINE, location=669ef94e8b26,44819,1731588865173; forceNewPlan=false, retain=false 2024-11-14T12:54:28,858 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1d33b9e9c25fcfc654d257646677381c, ASSIGN 2024-11-14T12:54:28,860 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=1d33b9e9c25fcfc654d257646677381c, ASSIGN; state=OFFLINE, location=669ef94e8b26,46877,1731588865107; forceNewPlan=false, retain=false 2024-11-14T12:54:28,863 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a5d884ba0737bc572475e022a6ed1bc8, ASSIGN 2024-11-14T12:54:28,870 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a5d884ba0737bc572475e022a6ed1bc8, ASSIGN; state=OFFLINE, location=669ef94e8b26,43475,1731588864980; forceNewPlan=false, retain=false 2024-11-14T12:54:28,876 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=18ddd925c95b1b23296976da04f97e83, ASSIGN 2024-11-14T12:54:28,879 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=18ddd925c95b1b23296976da04f97e83, ASSIGN; state=OFFLINE, location=669ef94e8b26,44819,1731588865173; forceNewPlan=false, retain=false 2024-11-14T12:54:28,882 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ea6e95809b33f853859fa227179bfe85, ASSIGN 2024-11-14T12:54:28,884 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ea6e95809b33f853859fa227179bfe85, ASSIGN; state=OFFLINE, location=669ef94e8b26,46877,1731588865107; forceNewPlan=false, retain=false 2024-11-14T12:54:28,886 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=338b47e35e9a8b9dd3045d365aaf47bd, ASSIGN 2024-11-14T12:54:28,889 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=338b47e35e9a8b9dd3045d365aaf47bd, ASSIGN; state=OFFLINE, location=669ef94e8b26,43475,1731588864980; forceNewPlan=false, retain=false 2024-11-14T12:54:28,890 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4f16209fa220cf1b7a18b891e4fe53c6, ASSIGN 2024-11-14T12:54:28,892 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4f16209fa220cf1b7a18b891e4fe53c6, ASSIGN; state=OFFLINE, location=669ef94e8b26,44819,1731588865173; forceNewPlan=false, retain=false 2024-11-14T12:54:28,896 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e90cc230c4f13fece446e3e8420dddbe, ASSIGN 2024-11-14T12:54:28,899 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=e90cc230c4f13fece446e3e8420dddbe, ASSIGN; state=OFFLINE, location=669ef94e8b26,46877,1731588865107; forceNewPlan=false, retain=false 2024-11-14T12:54:28,901 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=64b5bd4e6112417a55e871b36551c5b8, ASSIGN 2024-11-14T12:54:28,903 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=64b5bd4e6112417a55e871b36551c5b8, ASSIGN; state=OFFLINE, location=669ef94e8b26,43475,1731588864980; forceNewPlan=false, retain=false 2024-11-14T12:54:28,906 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=333a3eba2bb2d318c041beb4642f5ef3, ASSIGN 2024-11-14T12:54:28,908 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=333a3eba2bb2d318c041beb4642f5ef3, ASSIGN; state=OFFLINE, location=669ef94e8b26,44819,1731588865173; forceNewPlan=false, retain=false 2024-11-14T12:54:28,910 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1358fa84af33d1799bd11ba7f4959a37, ASSIGN 2024-11-14T12:54:28,912 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=1358fa84af33d1799bd11ba7f4959a37, ASSIGN; state=OFFLINE, location=669ef94e8b26,46877,1731588865107; forceNewPlan=false, retain=false 2024-11-14T12:54:28,914 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=03f289a2805f0b2f037581e9390ba191, ASSIGN 2024-11-14T12:54:28,915 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=03f289a2805f0b2f037581e9390ba191, ASSIGN; state=OFFLINE, location=669ef94e8b26,43475,1731588864980; forceNewPlan=false, retain=false 2024-11-14T12:54:28,917 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN 2024-11-14T12:54:28,919 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN; state=OFFLINE, location=669ef94e8b26,44819,1731588865173; forceNewPlan=false, retain=false 2024-11-14T12:54:28,921 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f76ec25f51f69b1106342a7dfae65fb6, ASSIGN 2024-11-14T12:54:28,923 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f76ec25f51f69b1106342a7dfae65fb6, ASSIGN; state=OFFLINE, location=669ef94e8b26,46877,1731588865107; forceNewPlan=false, retain=false 2024-11-14T12:54:28,924 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=bf47be17fd22bc204db28da07b30af2c, ASSIGN 2024-11-14T12:54:28,925 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=bf47be17fd22bc204db28da07b30af2c, ASSIGN; state=OFFLINE, location=669ef94e8b26,43475,1731588864980; forceNewPlan=false, retain=false 2024-11-14T12:54:28,927 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=23f7fbc112f1353900c1181e3ed36707, ASSIGN 2024-11-14T12:54:28,928 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=23f7fbc112f1353900c1181e3ed36707, ASSIGN; state=OFFLINE, location=669ef94e8b26,44819,1731588865173; forceNewPlan=false, retain=false 2024-11-14T12:54:28,930 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b8044551b940cf51bfa7c7633c4df21d, ASSIGN 2024-11-14T12:54:28,931 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=b8044551b940cf51bfa7c7633c4df21d, ASSIGN; state=OFFLINE, location=669ef94e8b26,43475,1731588864980; forceNewPlan=false, retain=false 2024-11-14T12:54:28,933 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0fd0512a8a2f36befd3453e1140df3af, ASSIGN 2024-11-14T12:54:28,935 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=0fd0512a8a2f36befd3453e1140df3af, ASSIGN; state=OFFLINE, location=669ef94e8b26,46877,1731588865107; forceNewPlan=false, retain=false 2024-11-14T12:54:28,937 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c2ccfc8c17ab1571bc4ea6d526017f9f, ASSIGN 2024-11-14T12:54:28,939 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c2ccfc8c17ab1571bc4ea6d526017f9f, ASSIGN; state=OFFLINE, location=669ef94e8b26,43475,1731588864980; forceNewPlan=false, retain=false 2024-11-14T12:54:28,941 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN 2024-11-14T12:54:28,943 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN; state=OFFLINE, location=669ef94e8b26,44819,1731588865173; forceNewPlan=false, retain=false 2024-11-14T12:54:28,977 INFO [669ef94e8b26:45309 {}] balancer.BaseLoadBalancer(388): Reassigned 26 regions. 26 retained the pre-restart assignment. 2024-11-14T12:54:28,978 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=cadcf21958eaec53ee489ca0918e3ab7, regionState=OPENING, regionLocation=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:28,979 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=87eb074dbd46b3bc6e27fb0bf3291d5b, regionState=OPENING, regionLocation=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:28,980 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=1d33b9e9c25fcfc654d257646677381c, regionState=OPENING, regionLocation=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:28,981 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=ea6e95809b33f853859fa227179bfe85, regionState=OPENING, regionLocation=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:28,982 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=e90cc230c4f13fece446e3e8420dddbe, regionState=OPENING, regionLocation=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:28,982 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=1358fa84af33d1799bd11ba7f4959a37, regionState=OPENING, regionLocation=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:28,983 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=f76ec25f51f69b1106342a7dfae65fb6, regionState=OPENING, regionLocation=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:28,984 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=0fd0512a8a2f36befd3453e1140df3af, regionState=OPENING, regionLocation=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:28,984 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=cadcf21958eaec53ee489ca0918e3ab7, ASSIGN because future has completed 2024-11-14T12:54:28,985 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=2201f29440b3aa3259741bbb374e209d, regionState=OPENING, regionLocation=669ef94e8b26,44819,1731588865173 2024-11-14T12:54:28,986 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=22b9f10374fc1ea7a876d476ce0b083d, regionState=OPENING, regionLocation=669ef94e8b26,44819,1731588865173 2024-11-14T12:54:28,986 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure cadcf21958eaec53ee489ca0918e3ab7, server=669ef94e8b26,46877,1731588865107}] 2024-11-14T12:54:28,986 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=87eb074dbd46b3bc6e27fb0bf3291d5b, ASSIGN because future has completed 2024-11-14T12:54:28,992 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=1d33b9e9c25fcfc654d257646677381c, ASSIGN because future has completed 2024-11-14T12:54:28,994 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1d33b9e9c25fcfc654d257646677381c, server=669ef94e8b26,46877,1731588865107}] 2024-11-14T12:54:28,995 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ea6e95809b33f853859fa227179bfe85, ASSIGN because future has completed 2024-11-14T12:54:28,997 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=e90cc230c4f13fece446e3e8420dddbe, ASSIGN because future has completed 2024-11-14T12:54:28,999 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=1358fa84af33d1799bd11ba7f4959a37, ASSIGN because future has completed 2024-11-14T12:54:29,001 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f76ec25f51f69b1106342a7dfae65fb6, ASSIGN because future has completed 2024-11-14T12:54:29,002 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure f76ec25f51f69b1106342a7dfae65fb6, server=669ef94e8b26,46877,1731588865107}] 2024-11-14T12:54:29,003 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=0fd0512a8a2f36befd3453e1140df3af, ASSIGN because future has completed 2024-11-14T12:54:29,009 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=2201f29440b3aa3259741bbb374e209d, ASSIGN because future has completed 2024-11-14T12:54:29,011 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=22b9f10374fc1ea7a876d476ce0b083d, ASSIGN because future has completed 2024-11-14T12:54:29,013 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 22b9f10374fc1ea7a876d476ce0b083d, server=669ef94e8b26,44819,1731588865173}] 2024-11-14T12:54:29,021 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2201f29440b3aa3259741bbb374e209d, server=669ef94e8b26,44819,1731588865173}] 2024-11-14T12:54:29,029 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0fd0512a8a2f36befd3453e1140df3af, server=669ef94e8b26,46877,1731588865107}] 2024-11-14T12:54:29,037 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1358fa84af33d1799bd11ba7f4959a37, server=669ef94e8b26,46877,1731588865107}] 2024-11-14T12:54:29,043 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure e90cc230c4f13fece446e3e8420dddbe, server=669ef94e8b26,46877,1731588865107}] 2024-11-14T12:54:29,049 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure ea6e95809b33f853859fa227179bfe85, server=669ef94e8b26,46877,1731588865107}] 2024-11-14T12:54:29,053 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 87eb074dbd46b3bc6e27fb0bf3291d5b, server=669ef94e8b26,46877,1731588865107}] 2024-11-14T12:54:29,058 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=c8ddc4f331afb3a3007ef5c49cdf6ab8, regionState=OPENING, regionLocation=669ef94e8b26,44819,1731588865173 2024-11-14T12:54:29,059 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=18ddd925c95b1b23296976da04f97e83, regionState=OPENING, regionLocation=669ef94e8b26,44819,1731588865173 2024-11-14T12:54:29,060 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=4f16209fa220cf1b7a18b891e4fe53c6, regionState=OPENING, regionLocation=669ef94e8b26,44819,1731588865173 2024-11-14T12:54:29,060 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=333a3eba2bb2d318c041beb4642f5ef3, regionState=OPENING, regionLocation=669ef94e8b26,44819,1731588865173 2024-11-14T12:54:29,061 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=cdc4b47ed9fe07b56101f9e0e17abcd7, regionState=OPENING, regionLocation=669ef94e8b26,44819,1731588865173 2024-11-14T12:54:29,061 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=23f7fbc112f1353900c1181e3ed36707, regionState=OPENING, regionLocation=669ef94e8b26,44819,1731588865173 2024-11-14T12:54:29,062 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=a572ae08c8738aa198c40f32aa4b5f96, regionState=OPENING, regionLocation=669ef94e8b26,44819,1731588865173 2024-11-14T12:54:29,063 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=66b34b9be40614bd68d0cea995e44193, regionState=OPENING, regionLocation=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:29,064 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=f8c0d62856832eec9f9ca20bb6db602c, regionState=OPENING, regionLocation=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:29,064 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN because future has completed 2024-11-14T12:54:29,065 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure c8ddc4f331afb3a3007ef5c49cdf6ab8, server=669ef94e8b26,44819,1731588865173}] 2024-11-14T12:54:29,066 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=18ddd925c95b1b23296976da04f97e83, ASSIGN because future has completed 2024-11-14T12:54:29,067 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4f16209fa220cf1b7a18b891e4fe53c6, ASSIGN because future has completed 2024-11-14T12:54:29,069 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=333a3eba2bb2d318c041beb4642f5ef3, ASSIGN because future has completed 2024-11-14T12:54:29,070 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN because future has completed 2024-11-14T12:54:29,071 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure cdc4b47ed9fe07b56101f9e0e17abcd7, server=669ef94e8b26,44819,1731588865173}] 2024-11-14T12:54:29,071 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=23f7fbc112f1353900c1181e3ed36707, ASSIGN because future has completed 2024-11-14T12:54:29,073 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN because future has completed 2024-11-14T12:54:29,074 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=66b34b9be40614bd68d0cea995e44193, ASSIGN because future has completed 2024-11-14T12:54:29,076 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f8c0d62856832eec9f9ca20bb6db602c, ASSIGN because future has completed 2024-11-14T12:54:29,078 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure f8c0d62856832eec9f9ca20bb6db602c, server=669ef94e8b26,43475,1731588864980}] 2024-11-14T12:54:29,083 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66b34b9be40614bd68d0cea995e44193, server=669ef94e8b26,43475,1731588864980}] 2024-11-14T12:54:29,088 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure a572ae08c8738aa198c40f32aa4b5f96, server=669ef94e8b26,44819,1731588865173}] 2024-11-14T12:54:29,093 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 23f7fbc112f1353900c1181e3ed36707, server=669ef94e8b26,44819,1731588865173}] 2024-11-14T12:54:29,097 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,44819,1731588865173}] 2024-11-14T12:54:29,102 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4f16209fa220cf1b7a18b891e4fe53c6, server=669ef94e8b26,44819,1731588865173}] 2024-11-14T12:54:29,108 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 18ddd925c95b1b23296976da04f97e83, server=669ef94e8b26,44819,1731588865173}] 2024-11-14T12:54:29,115 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=a5d884ba0737bc572475e022a6ed1bc8, regionState=OPENING, regionLocation=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:29,116 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=338b47e35e9a8b9dd3045d365aaf47bd, regionState=OPENING, regionLocation=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:29,117 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=64b5bd4e6112417a55e871b36551c5b8, regionState=OPENING, regionLocation=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:29,118 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=03f289a2805f0b2f037581e9390ba191, regionState=OPENING, regionLocation=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:29,119 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=bf47be17fd22bc204db28da07b30af2c, regionState=OPENING, regionLocation=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:29,119 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a5d884ba0737bc572475e022a6ed1bc8, ASSIGN because future has completed 2024-11-14T12:54:29,119 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=b8044551b940cf51bfa7c7633c4df21d, regionState=OPENING, regionLocation=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:29,120 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure a5d884ba0737bc572475e022a6ed1bc8, server=669ef94e8b26,43475,1731588864980}] 2024-11-14T12:54:29,123 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=338b47e35e9a8b9dd3045d365aaf47bd, ASSIGN because future has completed 2024-11-14T12:54:29,125 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=64b5bd4e6112417a55e871b36551c5b8, ASSIGN because future has completed 2024-11-14T12:54:29,127 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=03f289a2805f0b2f037581e9390ba191, ASSIGN because future has completed 2024-11-14T12:54:29,128 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 03f289a2805f0b2f037581e9390ba191, server=669ef94e8b26,43475,1731588864980}] 2024-11-14T12:54:29,128 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=bf47be17fd22bc204db28da07b30af2c, ASSIGN because future has completed 2024-11-14T12:54:29,130 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=b8044551b940cf51bfa7c7633c4df21d, ASSIGN because future has completed 2024-11-14T12:54:29,134 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure b8044551b940cf51bfa7c7633c4df21d, server=669ef94e8b26,43475,1731588864980}] 2024-11-14T12:54:29,141 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure bf47be17fd22bc204db28da07b30af2c, server=669ef94e8b26,43475,1731588864980}] 2024-11-14T12:54:29,141 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-14T12:54:29,143 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51613, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-14T12:54:29,149 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 64b5bd4e6112417a55e871b36551c5b8, server=669ef94e8b26,43475,1731588864980}] 2024-11-14T12:54:29,150 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open test,ggg,1731588868017.f76ec25f51f69b1106342a7dfae65fb6. 2024-11-14T12:54:29,150 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => f76ec25f51f69b1106342a7dfae65fb6, NAME => 'test,ggg,1731588868017.f76ec25f51f69b1106342a7dfae65fb6.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-14T12:54:29,150 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f76ec25f51f69b1106342a7dfae65fb6 2024-11-14T12:54:29,151 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated test,ggg,1731588868017.f76ec25f51f69b1106342a7dfae65fb6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:29,151 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for f76ec25f51f69b1106342a7dfae65fb6 2024-11-14T12:54:29,151 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for f76ec25f51f69b1106342a7dfae65fb6 2024-11-14T12:54:29,154 INFO [StoreOpener-f76ec25f51f69b1106342a7dfae65fb6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f76ec25f51f69b1106342a7dfae65fb6 2024-11-14T12:54:29,155 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 338b47e35e9a8b9dd3045d365aaf47bd, server=669ef94e8b26,43475,1731588864980}] 2024-11-14T12:54:29,157 INFO [StoreOpener-f76ec25f51f69b1106342a7dfae65fb6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f76ec25f51f69b1106342a7dfae65fb6 columnFamilyName family 2024-11-14T12:54:29,158 DEBUG [StoreOpener-f76ec25f51f69b1106342a7dfae65fb6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T12:54:29,159 INFO [StoreOpener-f76ec25f51f69b1106342a7dfae65fb6-1 {}] regionserver.HStore(327): Store=f76ec25f51f69b1106342a7dfae65fb6/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T12:54:29,160 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for f76ec25f51f69b1106342a7dfae65fb6 2024-11-14T12:54:29,161 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=c2ccfc8c17ab1571bc4ea6d526017f9f, regionState=OPENING, regionLocation=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:29,162 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/f76ec25f51f69b1106342a7dfae65fb6 2024-11-14T12:54:29,162 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/f76ec25f51f69b1106342a7dfae65fb6 2024-11-14T12:54:29,163 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for f76ec25f51f69b1106342a7dfae65fb6 2024-11-14T12:54:29,163 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for f76ec25f51f69b1106342a7dfae65fb6 2024-11-14T12:54:29,165 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c2ccfc8c17ab1571bc4ea6d526017f9f, ASSIGN because future has completed 2024-11-14T12:54:29,166 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for f76ec25f51f69b1106342a7dfae65fb6 2024-11-14T12:54:29,171 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure c2ccfc8c17ab1571bc4ea6d526017f9f, server=669ef94e8b26,43475,1731588864980}] 2024-11-14T12:54:29,173 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/f76ec25f51f69b1106342a7dfae65fb6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T12:54:29,175 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened f76ec25f51f69b1106342a7dfae65fb6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64439304, jitterRate=-0.03977954387664795}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T12:54:29,175 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f76ec25f51f69b1106342a7dfae65fb6 2024-11-14T12:54:29,177 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for f76ec25f51f69b1106342a7dfae65fb6: Running coprocessor pre-open hook at 1731588869151Writing region info on filesystem at 1731588869151Initializing all the Stores at 1731588869153 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731588869153Cleaning up temporary data from old regions at 1731588869163 (+10 ms)Running coprocessor post-open hooks at 1731588869175 (+12 ms)Region opened successfully at 1731588869177 (+2 ms) 2024-11-14T12:54:29,183 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ggg,1731588868017.f76ec25f51f69b1106342a7dfae65fb6., pid=33, masterSystemTime=1731588869140 2024-11-14T12:54:29,187 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ggg,1731588868017.f76ec25f51f69b1106342a7dfae65fb6. 2024-11-14T12:54:29,188 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened test,ggg,1731588868017.f76ec25f51f69b1106342a7dfae65fb6. 2024-11-14T12:54:29,188 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open test,jjj,1731588868017.1358fa84af33d1799bd11ba7f4959a37. 2024-11-14T12:54:29,188 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 1358fa84af33d1799bd11ba7f4959a37, NAME => 'test,jjj,1731588868017.1358fa84af33d1799bd11ba7f4959a37.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-14T12:54:29,188 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=f76ec25f51f69b1106342a7dfae65fb6, regionState=OPEN, openSeqNum=2, regionLocation=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:29,189 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 1358fa84af33d1799bd11ba7f4959a37 2024-11-14T12:54:29,189 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated test,jjj,1731588868017.1358fa84af33d1799bd11ba7f4959a37.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:29,189 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 1358fa84af33d1799bd11ba7f4959a37 2024-11-14T12:54:29,189 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 1358fa84af33d1799bd11ba7f4959a37 2024-11-14T12:54:29,192 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure f76ec25f51f69b1106342a7dfae65fb6, server=669ef94e8b26,46877,1731588865107 because future has completed 2024-11-14T12:54:29,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45309 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T12:54:29,194 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(132): Open test,nnn,1731588868017.4f16209fa220cf1b7a18b891e4fe53c6. 2024-11-14T12:54:29,194 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7752): Opening region: {ENCODED => 4f16209fa220cf1b7a18b891e4fe53c6, NAME => 'test,nnn,1731588868017.4f16209fa220cf1b7a18b891e4fe53c6.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-14T12:54:29,195 INFO [StoreOpener-1358fa84af33d1799bd11ba7f4959a37-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 1358fa84af33d1799bd11ba7f4959a37 2024-11-14T12:54:29,195 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4f16209fa220cf1b7a18b891e4fe53c6 2024-11-14T12:54:29,195 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(898): Instantiated test,nnn,1731588868017.4f16209fa220cf1b7a18b891e4fe53c6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:29,195 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7794): checking encryption for 4f16209fa220cf1b7a18b891e4fe53c6 2024-11-14T12:54:29,195 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7797): checking classloading for 4f16209fa220cf1b7a18b891e4fe53c6 2024-11-14T12:54:29,197 INFO [StoreOpener-1358fa84af33d1799bd11ba7f4959a37-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1358fa84af33d1799bd11ba7f4959a37 columnFamilyName family 2024-11-14T12:54:29,197 INFO [StoreOpener-4f16209fa220cf1b7a18b891e4fe53c6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4f16209fa220cf1b7a18b891e4fe53c6 2024-11-14T12:54:29,197 DEBUG [StoreOpener-1358fa84af33d1799bd11ba7f4959a37-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T12:54:29,198 INFO [StoreOpener-1358fa84af33d1799bd11ba7f4959a37-1 {}] regionserver.HStore(327): Store=1358fa84af33d1799bd11ba7f4959a37/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T12:54:29,198 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 1358fa84af33d1799bd11ba7f4959a37 2024-11-14T12:54:29,200 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/1358fa84af33d1799bd11ba7f4959a37 2024-11-14T12:54:29,200 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=11 2024-11-14T12:54:29,200 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure f76ec25f51f69b1106342a7dfae65fb6, server=669ef94e8b26,46877,1731588865107 in 194 msec 2024-11-14T12:54:29,200 INFO [StoreOpener-4f16209fa220cf1b7a18b891e4fe53c6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4f16209fa220cf1b7a18b891e4fe53c6 columnFamilyName family 2024-11-14T12:54:29,200 DEBUG [StoreOpener-4f16209fa220cf1b7a18b891e4fe53c6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T12:54:29,200 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/1358fa84af33d1799bd11ba7f4959a37 2024-11-14T12:54:29,201 INFO [StoreOpener-4f16209fa220cf1b7a18b891e4fe53c6-1 {}] regionserver.HStore(327): Store=4f16209fa220cf1b7a18b891e4fe53c6/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T12:54:29,201 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 1358fa84af33d1799bd11ba7f4959a37 2024-11-14T12:54:29,201 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 1358fa84af33d1799bd11ba7f4959a37 2024-11-14T12:54:29,201 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1038): replaying wal for 4f16209fa220cf1b7a18b891e4fe53c6 2024-11-14T12:54:29,203 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f76ec25f51f69b1106342a7dfae65fb6, ASSIGN in 391 msec 2024-11-14T12:54:29,204 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 1358fa84af33d1799bd11ba7f4959a37 2024-11-14T12:54:29,207 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/1358fa84af33d1799bd11ba7f4959a37/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T12:54:29,207 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 1358fa84af33d1799bd11ba7f4959a37; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64436668, jitterRate=-0.03981882333755493}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T12:54:29,208 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1358fa84af33d1799bd11ba7f4959a37 2024-11-14T12:54:29,208 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 1358fa84af33d1799bd11ba7f4959a37: Running coprocessor pre-open hook at 1731588869189Writing region info on filesystem at 1731588869189Initializing all the Stores at 1731588869191 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731588869191Cleaning up temporary data from old regions at 1731588869201 (+10 ms)Running coprocessor post-open hooks at 1731588869208 (+7 ms)Region opened successfully at 1731588869208 2024-11-14T12:54:29,209 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for test,jjj,1731588868017.1358fa84af33d1799bd11ba7f4959a37., pid=37, masterSystemTime=1731588869140 2024-11-14T12:54:29,212 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for test,jjj,1731588868017.1358fa84af33d1799bd11ba7f4959a37. 2024-11-14T12:54:29,212 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened test,jjj,1731588868017.1358fa84af33d1799bd11ba7f4959a37. 2024-11-14T12:54:29,212 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open test,sss,1731588868017.1d33b9e9c25fcfc654d257646677381c. 2024-11-14T12:54:29,213 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 1d33b9e9c25fcfc654d257646677381c, NAME => 'test,sss,1731588868017.1d33b9e9c25fcfc654d257646677381c.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-14T12:54:29,213 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=1358fa84af33d1799bd11ba7f4959a37, regionState=OPEN, openSeqNum=2, regionLocation=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:29,213 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 1d33b9e9c25fcfc654d257646677381c 2024-11-14T12:54:29,213 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated test,sss,1731588868017.1d33b9e9c25fcfc654d257646677381c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:29,214 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 1d33b9e9c25fcfc654d257646677381c 2024-11-14T12:54:29,214 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/4f16209fa220cf1b7a18b891e4fe53c6 2024-11-14T12:54:29,214 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 1d33b9e9c25fcfc654d257646677381c 2024-11-14T12:54:29,214 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/4f16209fa220cf1b7a18b891e4fe53c6 2024-11-14T12:54:29,215 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1048): stopping wal replay for 4f16209fa220cf1b7a18b891e4fe53c6 2024-11-14T12:54:29,215 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1060): Cleaning up temporary data for 4f16209fa220cf1b7a18b891e4fe53c6 2024-11-14T12:54:29,216 INFO [StoreOpener-1d33b9e9c25fcfc654d257646677381c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 1d33b9e9c25fcfc654d257646677381c 2024-11-14T12:54:29,218 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1093): writing seq id for 4f16209fa220cf1b7a18b891e4fe53c6 2024-11-14T12:54:29,219 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1358fa84af33d1799bd11ba7f4959a37, server=669ef94e8b26,46877,1731588865107 because future has completed 2024-11-14T12:54:29,220 INFO [StoreOpener-1d33b9e9c25fcfc654d257646677381c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1d33b9e9c25fcfc654d257646677381c columnFamilyName family 2024-11-14T12:54:29,220 DEBUG [StoreOpener-1d33b9e9c25fcfc654d257646677381c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T12:54:29,221 INFO [StoreOpener-1d33b9e9c25fcfc654d257646677381c-1 {}] regionserver.HStore(327): Store=1d33b9e9c25fcfc654d257646677381c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T12:54:29,222 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 1d33b9e9c25fcfc654d257646677381c 2024-11-14T12:54:29,223 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/4f16209fa220cf1b7a18b891e4fe53c6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T12:54:29,224 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/1d33b9e9c25fcfc654d257646677381c 2024-11-14T12:54:29,225 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1114): Opened 4f16209fa220cf1b7a18b891e4fe53c6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60835512, jitterRate=-0.09348022937774658}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T12:54:29,225 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4f16209fa220cf1b7a18b891e4fe53c6 2024-11-14T12:54:29,225 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/1d33b9e9c25fcfc654d257646677381c 2024-11-14T12:54:29,225 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1006): Region open journal for 4f16209fa220cf1b7a18b891e4fe53c6: Running coprocessor pre-open hook at 1731588869195Writing region info on filesystem at 1731588869195Initializing all the Stores at 1731588869197 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731588869197Cleaning up temporary data from old regions at 1731588869215 (+18 ms)Running coprocessor post-open hooks at 1731588869225 (+10 ms)Region opened successfully at 1731588869225 2024-11-14T12:54:29,227 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=14 2024-11-14T12:54:29,227 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2236): Post open deploy tasks for test,nnn,1731588868017.4f16209fa220cf1b7a18b891e4fe53c6., pid=48, masterSystemTime=1731588869168 2024-11-14T12:54:29,227 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 1358fa84af33d1799bd11ba7f4959a37, server=669ef94e8b26,46877,1731588865107 in 185 msec 2024-11-14T12:54:29,227 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 1d33b9e9c25fcfc654d257646677381c 2024-11-14T12:54:29,227 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 1d33b9e9c25fcfc654d257646677381c 2024-11-14T12:54:29,230 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2266): Finished post open deploy task for test,nnn,1731588868017.4f16209fa220cf1b7a18b891e4fe53c6. 2024-11-14T12:54:29,230 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(153): Opened test,nnn,1731588868017.4f16209fa220cf1b7a18b891e4fe53c6. 2024-11-14T12:54:29,231 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(132): Open test,qqq,1731588868017.18ddd925c95b1b23296976da04f97e83. 2024-11-14T12:54:29,231 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-14T12:54:29,231 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7752): Opening region: {ENCODED => 18ddd925c95b1b23296976da04f97e83, NAME => 'test,qqq,1731588868017.18ddd925c95b1b23296976da04f97e83.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-11-14T12:54:29,231 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=1358fa84af33d1799bd11ba7f4959a37, ASSIGN in 418 msec 2024-11-14T12:54:29,231 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 18ddd925c95b1b23296976da04f97e83 2024-11-14T12:54:29,231 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(898): Instantiated test,qqq,1731588868017.18ddd925c95b1b23296976da04f97e83.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:29,232 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7794): checking encryption for 18ddd925c95b1b23296976da04f97e83 2024-11-14T12:54:29,232 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7797): checking classloading for 18ddd925c95b1b23296976da04f97e83 2024-11-14T12:54:29,233 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 1d33b9e9c25fcfc654d257646677381c 2024-11-14T12:54:29,233 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=4f16209fa220cf1b7a18b891e4fe53c6, regionState=OPEN, openSeqNum=2, regionLocation=669ef94e8b26,44819,1731588865173 2024-11-14T12:54:29,233 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41689, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-14T12:54:29,235 INFO [StoreOpener-18ddd925c95b1b23296976da04f97e83-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 18ddd925c95b1b23296976da04f97e83 2024-11-14T12:54:29,239 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=48, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4f16209fa220cf1b7a18b891e4fe53c6, server=669ef94e8b26,44819,1731588865173 because future has completed 2024-11-14T12:54:29,240 INFO [StoreOpener-18ddd925c95b1b23296976da04f97e83-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 18ddd925c95b1b23296976da04f97e83 columnFamilyName family 2024-11-14T12:54:29,240 DEBUG [StoreOpener-18ddd925c95b1b23296976da04f97e83-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T12:54:29,241 INFO [StoreOpener-18ddd925c95b1b23296976da04f97e83-1 {}] regionserver.HStore(327): Store=18ddd925c95b1b23296976da04f97e83/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T12:54:29,242 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1038): replaying wal for 18ddd925c95b1b23296976da04f97e83 2024-11-14T12:54:29,243 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/18ddd925c95b1b23296976da04f97e83 2024-11-14T12:54:29,244 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/18ddd925c95b1b23296976da04f97e83 2024-11-14T12:54:29,245 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1048): stopping wal replay for 18ddd925c95b1b23296976da04f97e83 2024-11-14T12:54:29,245 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1060): Cleaning up temporary data for 18ddd925c95b1b23296976da04f97e83 2024-11-14T12:54:29,247 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=18 2024-11-14T12:54:29,247 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure 4f16209fa220cf1b7a18b891e4fe53c6, server=669ef94e8b26,44819,1731588865173 in 140 msec 2024-11-14T12:54:29,250 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): writing seq id for 18ddd925c95b1b23296976da04f97e83 2024-11-14T12:54:29,253 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=4f16209fa220cf1b7a18b891e4fe53c6, ASSIGN in 439 msec 2024-11-14T12:54:29,255 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/1d33b9e9c25fcfc654d257646677381c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T12:54:29,260 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 1d33b9e9c25fcfc654d257646677381c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69009942, jitterRate=0.028328269720077515}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T12:54:29,260 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1d33b9e9c25fcfc654d257646677381c 2024-11-14T12:54:29,260 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 1d33b9e9c25fcfc654d257646677381c: Running coprocessor pre-open hook at 1731588869214Writing region info on filesystem at 1731588869214Initializing all the Stores at 1731588869215 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731588869215Cleaning up temporary data from old regions at 1731588869227 (+12 ms)Running coprocessor post-open hooks at 1731588869260 (+33 ms)Region opened successfully at 1731588869260 2024-11-14T12:54:29,261 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/18ddd925c95b1b23296976da04f97e83/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T12:54:29,262 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1731588868017.1d33b9e9c25fcfc654d257646677381c., pid=32, masterSystemTime=1731588869140 2024-11-14T12:54:29,263 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1114): Opened 18ddd925c95b1b23296976da04f97e83; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64292793, jitterRate=-0.04196272790431976}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T12:54:29,263 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 18ddd925c95b1b23296976da04f97e83 2024-11-14T12:54:29,263 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1006): Region open journal for 18ddd925c95b1b23296976da04f97e83: Running coprocessor pre-open hook at 1731588869232Writing region info on filesystem at 1731588869232Initializing all the Stores at 1731588869234 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731588869234Cleaning up temporary data from old regions at 1731588869245 (+11 ms)Running coprocessor post-open hooks at 1731588869263 (+18 ms)Region opened successfully at 1731588869263 2024-11-14T12:54:29,264 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(132): Open test,lll,1731588868017.64b5bd4e6112417a55e871b36551c5b8. 2024-11-14T12:54:29,265 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7752): Opening region: {ENCODED => 64b5bd4e6112417a55e871b36551c5b8, NAME => 'test,lll,1731588868017.64b5bd4e6112417a55e871b36551c5b8.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-14T12:54:29,265 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2236): Post open deploy tasks for test,qqq,1731588868017.18ddd925c95b1b23296976da04f97e83., pid=49, masterSystemTime=1731588869168 2024-11-14T12:54:29,266 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 64b5bd4e6112417a55e871b36551c5b8 2024-11-14T12:54:29,266 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(898): Instantiated test,lll,1731588868017.64b5bd4e6112417a55e871b36551c5b8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:29,267 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1731588868017.1d33b9e9c25fcfc654d257646677381c. 2024-11-14T12:54:29,267 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened test,sss,1731588868017.1d33b9e9c25fcfc654d257646677381c. 2024-11-14T12:54:29,267 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open test,ccc,1731588868017.0fd0512a8a2f36befd3453e1140df3af. 2024-11-14T12:54:29,267 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7794): checking encryption for 64b5bd4e6112417a55e871b36551c5b8 2024-11-14T12:54:29,267 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7797): checking classloading for 64b5bd4e6112417a55e871b36551c5b8 2024-11-14T12:54:29,267 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => 0fd0512a8a2f36befd3453e1140df3af, NAME => 'test,ccc,1731588868017.0fd0512a8a2f36befd3453e1140df3af.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-14T12:54:29,268 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 0fd0512a8a2f36befd3453e1140df3af 2024-11-14T12:54:29,268 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated test,ccc,1731588868017.0fd0512a8a2f36befd3453e1140df3af.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:29,268 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for 0fd0512a8a2f36befd3453e1140df3af 2024-11-14T12:54:29,268 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for 0fd0512a8a2f36befd3453e1140df3af 2024-11-14T12:54:29,269 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=1d33b9e9c25fcfc654d257646677381c, regionState=OPEN, openSeqNum=2, regionLocation=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:29,270 INFO [StoreOpener-64b5bd4e6112417a55e871b36551c5b8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 64b5bd4e6112417a55e871b36551c5b8 2024-11-14T12:54:29,270 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2266): Finished post open deploy task for test,qqq,1731588868017.18ddd925c95b1b23296976da04f97e83. 2024-11-14T12:54:29,270 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(153): Opened test,qqq,1731588868017.18ddd925c95b1b23296976da04f97e83. 2024-11-14T12:54:29,270 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open test,zzz,1731588868017.2201f29440b3aa3259741bbb374e209d. 2024-11-14T12:54:29,271 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => 2201f29440b3aa3259741bbb374e209d, NAME => 'test,zzz,1731588868017.2201f29440b3aa3259741bbb374e209d.', STARTKEY => 'zzz', ENDKEY => ''} 2024-11-14T12:54:29,271 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 2201f29440b3aa3259741bbb374e209d 2024-11-14T12:54:29,271 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated test,zzz,1731588868017.2201f29440b3aa3259741bbb374e209d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:29,272 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for 2201f29440b3aa3259741bbb374e209d 2024-11-14T12:54:29,272 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for 2201f29440b3aa3259741bbb374e209d 2024-11-14T12:54:29,273 INFO [StoreOpener-64b5bd4e6112417a55e871b36551c5b8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 64b5bd4e6112417a55e871b36551c5b8 columnFamilyName family 2024-11-14T12:54:29,273 DEBUG [StoreOpener-64b5bd4e6112417a55e871b36551c5b8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T12:54:29,276 INFO [StoreOpener-0fd0512a8a2f36befd3453e1140df3af-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 0fd0512a8a2f36befd3453e1140df3af 2024-11-14T12:54:29,278 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1d33b9e9c25fcfc654d257646677381c, server=669ef94e8b26,46877,1731588865107 because future has completed 2024-11-14T12:54:29,278 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=18ddd925c95b1b23296976da04f97e83, regionState=OPEN, openSeqNum=2, regionLocation=669ef94e8b26,44819,1731588865173 2024-11-14T12:54:29,278 INFO [StoreOpener-64b5bd4e6112417a55e871b36551c5b8-1 {}] regionserver.HStore(327): Store=64b5bd4e6112417a55e871b36551c5b8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T12:54:29,279 INFO [StoreOpener-0fd0512a8a2f36befd3453e1140df3af-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0fd0512a8a2f36befd3453e1140df3af columnFamilyName family 2024-11-14T12:54:29,279 DEBUG [StoreOpener-0fd0512a8a2f36befd3453e1140df3af-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T12:54:29,281 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1038): replaying wal for 64b5bd4e6112417a55e871b36551c5b8 2024-11-14T12:54:29,281 INFO [StoreOpener-0fd0512a8a2f36befd3453e1140df3af-1 {}] regionserver.HStore(327): Store=0fd0512a8a2f36befd3453e1140df3af/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T12:54:29,281 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for 0fd0512a8a2f36befd3453e1140df3af 2024-11-14T12:54:29,283 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/0fd0512a8a2f36befd3453e1140df3af 2024-11-14T12:54:29,283 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 18ddd925c95b1b23296976da04f97e83, server=669ef94e8b26,44819,1731588865173 because future has completed 2024-11-14T12:54:29,284 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/64b5bd4e6112417a55e871b36551c5b8 2024-11-14T12:54:29,284 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/0fd0512a8a2f36befd3453e1140df3af 2024-11-14T12:54:29,285 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for 0fd0512a8a2f36befd3453e1140df3af 2024-11-14T12:54:29,285 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/64b5bd4e6112417a55e871b36551c5b8 2024-11-14T12:54:29,285 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for 0fd0512a8a2f36befd3453e1140df3af 2024-11-14T12:54:29,285 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1048): stopping wal replay for 64b5bd4e6112417a55e871b36551c5b8 2024-11-14T12:54:29,285 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1060): Cleaning up temporary data for 64b5bd4e6112417a55e871b36551c5b8 2024-11-14T12:54:29,286 INFO [StoreOpener-2201f29440b3aa3259741bbb374e209d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 2201f29440b3aa3259741bbb374e209d 2024-11-14T12:54:29,289 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1093): writing seq id for 64b5bd4e6112417a55e871b36551c5b8 2024-11-14T12:54:29,290 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for 0fd0512a8a2f36befd3453e1140df3af 2024-11-14T12:54:29,290 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=23 2024-11-14T12:54:29,290 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 1d33b9e9c25fcfc654d257646677381c, server=669ef94e8b26,46877,1731588865107 in 290 msec 2024-11-14T12:54:29,291 INFO [StoreOpener-2201f29440b3aa3259741bbb374e209d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2201f29440b3aa3259741bbb374e209d columnFamilyName family 2024-11-14T12:54:29,291 DEBUG [StoreOpener-2201f29440b3aa3259741bbb374e209d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T12:54:29,292 INFO [StoreOpener-2201f29440b3aa3259741bbb374e209d-1 {}] regionserver.HStore(327): Store=2201f29440b3aa3259741bbb374e209d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T12:54:29,293 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for 2201f29440b3aa3259741bbb374e209d 2024-11-14T12:54:29,294 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=1d33b9e9c25fcfc654d257646677381c, ASSIGN in 481 msec 2024-11-14T12:54:29,295 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/2201f29440b3aa3259741bbb374e209d 2024-11-14T12:54:29,295 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/64b5bd4e6112417a55e871b36551c5b8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T12:54:29,296 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/2201f29440b3aa3259741bbb374e209d 2024-11-14T12:54:29,296 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1114): Opened 64b5bd4e6112417a55e871b36551c5b8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72678579, jitterRate=0.0829952210187912}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T12:54:29,296 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 64b5bd4e6112417a55e871b36551c5b8 2024-11-14T12:54:29,296 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1006): Region open journal for 64b5bd4e6112417a55e871b36551c5b8: Running coprocessor pre-open hook at 1731588869267Writing region info on filesystem at 1731588869267Initializing all the Stores at 1731588869269 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731588869269Cleaning up temporary data from old regions at 1731588869285 (+16 ms)Running coprocessor post-open hooks at 1731588869296 (+11 ms)Region opened successfully at 1731588869296 2024-11-14T12:54:29,297 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for 2201f29440b3aa3259741bbb374e209d 2024-11-14T12:54:29,297 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/0fd0512a8a2f36befd3453e1140df3af/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T12:54:29,297 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for 2201f29440b3aa3259741bbb374e209d 2024-11-14T12:54:29,298 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1731588868017.64b5bd4e6112417a55e871b36551c5b8., pid=54, masterSystemTime=1731588869230 2024-11-14T12:54:29,298 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened 0fd0512a8a2f36befd3453e1140df3af; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59246377, jitterRate=-0.11716018617153168}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T12:54:29,298 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0fd0512a8a2f36befd3453e1140df3af 2024-11-14T12:54:29,298 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for 0fd0512a8a2f36befd3453e1140df3af: Running coprocessor pre-open hook at 1731588869268Writing region info on filesystem at 1731588869268Initializing all the Stores at 1731588869270 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731588869270Cleaning up temporary data from old regions at 1731588869285 (+15 ms)Running coprocessor post-open hooks at 1731588869298 (+13 ms)Region opened successfully at 1731588869298 2024-11-14T12:54:29,301 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1731588868017.0fd0512a8a2f36befd3453e1140df3af., pid=36, masterSystemTime=1731588869140 2024-11-14T12:54:29,303 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for 2201f29440b3aa3259741bbb374e209d 2024-11-14T12:54:29,303 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1731588868017.64b5bd4e6112417a55e871b36551c5b8. 2024-11-14T12:54:29,304 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(153): Opened test,lll,1731588868017.64b5bd4e6112417a55e871b36551c5b8. 2024-11-14T12:54:29,304 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open test,fff,1731588868017.bf47be17fd22bc204db28da07b30af2c. 2024-11-14T12:54:29,304 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => bf47be17fd22bc204db28da07b30af2c, NAME => 'test,fff,1731588868017.bf47be17fd22bc204db28da07b30af2c.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-14T12:54:29,304 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test bf47be17fd22bc204db28da07b30af2c 2024-11-14T12:54:29,305 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated test,fff,1731588868017.bf47be17fd22bc204db28da07b30af2c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:29,305 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for bf47be17fd22bc204db28da07b30af2c 2024-11-14T12:54:29,305 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for bf47be17fd22bc204db28da07b30af2c 2024-11-14T12:54:29,305 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1731588868017.0fd0512a8a2f36befd3453e1140df3af. 2024-11-14T12:54:29,306 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened test,ccc,1731588868017.0fd0512a8a2f36befd3453e1140df3af. 2024-11-14T12:54:29,306 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open test,ppp,1731588868017.ea6e95809b33f853859fa227179bfe85. 2024-11-14T12:54:29,306 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => ea6e95809b33f853859fa227179bfe85, NAME => 'test,ppp,1731588868017.ea6e95809b33f853859fa227179bfe85.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-14T12:54:29,307 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ea6e95809b33f853859fa227179bfe85 2024-11-14T12:54:29,307 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated test,ppp,1731588868017.ea6e95809b33f853859fa227179bfe85.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:29,307 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for ea6e95809b33f853859fa227179bfe85 2024-11-14T12:54:29,307 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for ea6e95809b33f853859fa227179bfe85 2024-11-14T12:54:29,307 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=21 2024-11-14T12:54:29,307 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=21, state=SUCCESS, hasLock=false; OpenRegionProcedure 18ddd925c95b1b23296976da04f97e83, server=669ef94e8b26,44819,1731588865173 in 190 msec 2024-11-14T12:54:29,311 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=18ddd925c95b1b23296976da04f97e83, ASSIGN in 499 msec 2024-11-14T12:54:29,313 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/2201f29440b3aa3259741bbb374e209d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T12:54:29,313 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=0fd0512a8a2f36befd3453e1140df3af, regionState=OPEN, openSeqNum=2, regionLocation=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:29,314 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened 2201f29440b3aa3259741bbb374e209d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59370492, jitterRate=-0.11531072854995728}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T12:54:29,314 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2201f29440b3aa3259741bbb374e209d 2024-11-14T12:54:29,314 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for 2201f29440b3aa3259741bbb374e209d: Running coprocessor pre-open hook at 1731588869272Writing region info on filesystem at 1731588869272Initializing all the Stores at 1731588869278 (+6 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731588869278Cleaning up temporary data from old regions at 1731588869297 (+19 ms)Running coprocessor post-open hooks at 1731588869314 (+17 ms)Region opened successfully at 1731588869314 2024-11-14T12:54:29,316 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for test,zzz,1731588868017.2201f29440b3aa3259741bbb374e209d., pid=35, masterSystemTime=1731588869168 2024-11-14T12:54:29,318 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=64b5bd4e6112417a55e871b36551c5b8, regionState=OPEN, openSeqNum=2, regionLocation=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:29,318 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0fd0512a8a2f36befd3453e1140df3af, server=669ef94e8b26,46877,1731588865107 because future has completed 2024-11-14T12:54:29,321 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for test,zzz,1731588868017.2201f29440b3aa3259741bbb374e209d. 2024-11-14T12:54:29,321 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened test,zzz,1731588868017.2201f29440b3aa3259741bbb374e209d. 2024-11-14T12:54:29,322 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(132): Open test,hhh,1731588868017.cdc4b47ed9fe07b56101f9e0e17abcd7. 2024-11-14T12:54:29,322 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7752): Opening region: {ENCODED => cdc4b47ed9fe07b56101f9e0e17abcd7, NAME => 'test,hhh,1731588868017.cdc4b47ed9fe07b56101f9e0e17abcd7.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-14T12:54:29,323 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test cdc4b47ed9fe07b56101f9e0e17abcd7 2024-11-14T12:54:29,323 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 64b5bd4e6112417a55e871b36551c5b8, server=669ef94e8b26,43475,1731588864980 because future has completed 2024-11-14T12:54:29,323 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(898): Instantiated test,hhh,1731588868017.cdc4b47ed9fe07b56101f9e0e17abcd7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:29,324 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7794): checking encryption for cdc4b47ed9fe07b56101f9e0e17abcd7 2024-11-14T12:54:29,324 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7797): checking classloading for cdc4b47ed9fe07b56101f9e0e17abcd7 2024-11-14T12:54:29,325 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=2201f29440b3aa3259741bbb374e209d, regionState=OPEN, openSeqNum=2, regionLocation=669ef94e8b26,44819,1731588865173 2024-11-14T12:54:29,329 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45309 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=669ef94e8b26,43475,1731588864980, table=test, region=64b5bd4e6112417a55e871b36551c5b8. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-14T12:54:29,329 INFO [StoreOpener-bf47be17fd22bc204db28da07b30af2c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region bf47be17fd22bc204db28da07b30af2c 2024-11-14T12:54:29,331 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2201f29440b3aa3259741bbb374e209d, server=669ef94e8b26,44819,1731588865173 because future has completed 2024-11-14T12:54:29,331 INFO [StoreOpener-ea6e95809b33f853859fa227179bfe85-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ea6e95809b33f853859fa227179bfe85 2024-11-14T12:54:29,332 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45309 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=669ef94e8b26,46877,1731588865107, table=test, region=0fd0512a8a2f36befd3453e1140df3af. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-14T12:54:29,333 INFO [StoreOpener-ea6e95809b33f853859fa227179bfe85-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea6e95809b33f853859fa227179bfe85 columnFamilyName family 2024-11-14T12:54:29,333 INFO [StoreOpener-bf47be17fd22bc204db28da07b30af2c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bf47be17fd22bc204db28da07b30af2c columnFamilyName family 2024-11-14T12:54:29,333 DEBUG [StoreOpener-ea6e95809b33f853859fa227179bfe85-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T12:54:29,333 DEBUG [StoreOpener-bf47be17fd22bc204db28da07b30af2c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T12:54:29,334 INFO [StoreOpener-bf47be17fd22bc204db28da07b30af2c-1 {}] regionserver.HStore(327): Store=bf47be17fd22bc204db28da07b30af2c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T12:54:29,334 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for bf47be17fd22bc204db28da07b30af2c 2024-11-14T12:54:29,335 INFO [StoreOpener-ea6e95809b33f853859fa227179bfe85-1 {}] regionserver.HStore(327): Store=ea6e95809b33f853859fa227179bfe85/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T12:54:29,335 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for ea6e95809b33f853859fa227179bfe85 2024-11-14T12:54:29,336 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/bf47be17fd22bc204db28da07b30af2c 2024-11-14T12:54:29,336 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/ea6e95809b33f853859fa227179bfe85 2024-11-14T12:54:29,336 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/bf47be17fd22bc204db28da07b30af2c 2024-11-14T12:54:29,337 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for bf47be17fd22bc204db28da07b30af2c 2024-11-14T12:54:29,338 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for bf47be17fd22bc204db28da07b30af2c 2024-11-14T12:54:29,338 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/ea6e95809b33f853859fa227179bfe85 2024-11-14T12:54:29,338 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for ea6e95809b33f853859fa227179bfe85 2024-11-14T12:54:29,339 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for ea6e95809b33f853859fa227179bfe85 2024-11-14T12:54:29,341 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for bf47be17fd22bc204db28da07b30af2c 2024-11-14T12:54:29,342 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for ea6e95809b33f853859fa227179bfe85 2024-11-14T12:54:29,342 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=16 2024-11-14T12:54:29,342 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 64b5bd4e6112417a55e871b36551c5b8, server=669ef94e8b26,43475,1731588864980 in 186 msec 2024-11-14T12:54:29,348 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=64b5bd4e6112417a55e871b36551c5b8, ASSIGN in 534 msec 2024-11-14T12:54:29,351 INFO [StoreOpener-cdc4b47ed9fe07b56101f9e0e17abcd7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region cdc4b47ed9fe07b56101f9e0e17abcd7 2024-11-14T12:54:29,354 INFO [StoreOpener-cdc4b47ed9fe07b56101f9e0e17abcd7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cdc4b47ed9fe07b56101f9e0e17abcd7 columnFamilyName family 2024-11-14T12:54:29,354 DEBUG [StoreOpener-cdc4b47ed9fe07b56101f9e0e17abcd7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T12:54:29,355 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=30 2024-11-14T12:54:29,355 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure 2201f29440b3aa3259741bbb374e209d, server=669ef94e8b26,44819,1731588865173 in 328 msec 2024-11-14T12:54:29,355 INFO [StoreOpener-cdc4b47ed9fe07b56101f9e0e17abcd7-1 {}] regionserver.HStore(327): Store=cdc4b47ed9fe07b56101f9e0e17abcd7/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T12:54:29,355 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1038): replaying wal for cdc4b47ed9fe07b56101f9e0e17abcd7 2024-11-14T12:54:29,357 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/cdc4b47ed9fe07b56101f9e0e17abcd7 2024-11-14T12:54:29,358 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/cdc4b47ed9fe07b56101f9e0e17abcd7 2024-11-14T12:54:29,358 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=2201f29440b3aa3259741bbb374e209d, ASSIGN in 546 msec 2024-11-14T12:54:29,358 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1048): stopping wal replay for cdc4b47ed9fe07b56101f9e0e17abcd7 2024-11-14T12:54:29,359 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1060): Cleaning up temporary data for cdc4b47ed9fe07b56101f9e0e17abcd7 2024-11-14T12:54:29,360 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/bf47be17fd22bc204db28da07b30af2c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T12:54:29,362 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened bf47be17fd22bc204db28da07b30af2c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72110291, jitterRate=0.0745270699262619}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T12:54:29,362 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bf47be17fd22bc204db28da07b30af2c 2024-11-14T12:54:29,362 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for bf47be17fd22bc204db28da07b30af2c: Running coprocessor pre-open hook at 1731588869305Writing region info on filesystem at 1731588869305Initializing all the Stores at 1731588869308 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731588869308Cleaning up temporary data from old regions at 1731588869338 (+30 ms)Running coprocessor post-open hooks at 1731588869362 (+24 ms)Region opened successfully at 1731588869362 2024-11-14T12:54:29,363 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1731588868017.bf47be17fd22bc204db28da07b30af2c., pid=53, masterSystemTime=1731588869230 2024-11-14T12:54:29,363 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1093): writing seq id for cdc4b47ed9fe07b56101f9e0e17abcd7 2024-11-14T12:54:29,366 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/ea6e95809b33f853859fa227179bfe85/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T12:54:29,367 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1731588868017.bf47be17fd22bc204db28da07b30af2c. 2024-11-14T12:54:29,367 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened test,fff,1731588868017.bf47be17fd22bc204db28da07b30af2c. 2024-11-14T12:54:29,367 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open test,ooo,1731588868017.338b47e35e9a8b9dd3045d365aaf47bd. 2024-11-14T12:54:29,367 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => 338b47e35e9a8b9dd3045d365aaf47bd, NAME => 'test,ooo,1731588868017.338b47e35e9a8b9dd3045d365aaf47bd.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-11-14T12:54:29,368 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/cdc4b47ed9fe07b56101f9e0e17abcd7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T12:54:29,368 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 338b47e35e9a8b9dd3045d365aaf47bd 2024-11-14T12:54:29,368 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated test,ooo,1731588868017.338b47e35e9a8b9dd3045d365aaf47bd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:29,368 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for 338b47e35e9a8b9dd3045d365aaf47bd 2024-11-14T12:54:29,368 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for 338b47e35e9a8b9dd3045d365aaf47bd 2024-11-14T12:54:29,368 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened ea6e95809b33f853859fa227179bfe85; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72099451, jitterRate=0.0743655413389206}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T12:54:29,368 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ea6e95809b33f853859fa227179bfe85 2024-11-14T12:54:29,368 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1114): Opened cdc4b47ed9fe07b56101f9e0e17abcd7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63261440, jitterRate=-0.057331085205078125}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T12:54:29,368 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cdc4b47ed9fe07b56101f9e0e17abcd7 2024-11-14T12:54:29,369 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1006): Region open journal for cdc4b47ed9fe07b56101f9e0e17abcd7: Running coprocessor pre-open hook at 1731588869324Writing region info on filesystem at 1731588869324Initializing all the Stores at 1731588869327 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731588869327Cleaning up temporary data from old regions at 1731588869359 (+32 ms)Running coprocessor post-open hooks at 1731588869369 (+10 ms)Region opened successfully at 1731588869369 2024-11-14T12:54:29,368 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for ea6e95809b33f853859fa227179bfe85: Running coprocessor pre-open hook at 1731588869307Writing region info on filesystem at 1731588869307Initializing all the Stores at 1731588869309 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731588869309Cleaning up temporary data from old regions at 1731588869339 (+30 ms)Running coprocessor post-open hooks at 1731588869368 (+29 ms)Region opened successfully at 1731588869368 2024-11-14T12:54:29,370 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1731588868017.cdc4b47ed9fe07b56101f9e0e17abcd7., pid=42, masterSystemTime=1731588869168 2024-11-14T12:54:29,370 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=7 2024-11-14T12:54:29,371 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 0fd0512a8a2f36befd3453e1140df3af, server=669ef94e8b26,46877,1731588865107 in 332 msec 2024-11-14T12:54:29,371 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1731588868017.ea6e95809b33f853859fa227179bfe85., pid=39, masterSystemTime=1731588869140 2024-11-14T12:54:29,371 INFO [StoreOpener-338b47e35e9a8b9dd3045d365aaf47bd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 338b47e35e9a8b9dd3045d365aaf47bd 2024-11-14T12:54:29,373 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1731588868017.cdc4b47ed9fe07b56101f9e0e17abcd7. 2024-11-14T12:54:29,374 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(153): Opened test,hhh,1731588868017.cdc4b47ed9fe07b56101f9e0e17abcd7. 2024-11-14T12:54:29,374 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open test,kkk,1731588868017.333a3eba2bb2d318c041beb4642f5ef3. 2024-11-14T12:54:29,374 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => 333a3eba2bb2d318c041beb4642f5ef3, NAME => 'test,kkk,1731588868017.333a3eba2bb2d318c041beb4642f5ef3.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-11-14T12:54:29,374 INFO [StoreOpener-338b47e35e9a8b9dd3045d365aaf47bd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 338b47e35e9a8b9dd3045d365aaf47bd columnFamilyName family 2024-11-14T12:54:29,374 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 333a3eba2bb2d318c041beb4642f5ef3 2024-11-14T12:54:29,375 DEBUG [StoreOpener-338b47e35e9a8b9dd3045d365aaf47bd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T12:54:29,375 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated test,kkk,1731588868017.333a3eba2bb2d318c041beb4642f5ef3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:29,375 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=0fd0512a8a2f36befd3453e1140df3af, ASSIGN in 562 msec 2024-11-14T12:54:29,375 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for 333a3eba2bb2d318c041beb4642f5ef3 2024-11-14T12:54:29,375 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for 333a3eba2bb2d318c041beb4642f5ef3 2024-11-14T12:54:29,375 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ppp,1731588868017.ea6e95809b33f853859fa227179bfe85. 2024-11-14T12:54:29,375 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened test,ppp,1731588868017.ea6e95809b33f853859fa227179bfe85. 2024-11-14T12:54:29,375 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open test,mmm,1731588868017.e90cc230c4f13fece446e3e8420dddbe. 2024-11-14T12:54:29,376 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => e90cc230c4f13fece446e3e8420dddbe, NAME => 'test,mmm,1731588868017.e90cc230c4f13fece446e3e8420dddbe.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-14T12:54:29,376 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test e90cc230c4f13fece446e3e8420dddbe 2024-11-14T12:54:29,376 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated test,mmm,1731588868017.e90cc230c4f13fece446e3e8420dddbe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:29,376 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for e90cc230c4f13fece446e3e8420dddbe 2024-11-14T12:54:29,376 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for e90cc230c4f13fece446e3e8420dddbe 2024-11-14T12:54:29,377 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=ea6e95809b33f853859fa227179bfe85, regionState=OPEN, openSeqNum=2, regionLocation=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:29,377 INFO [StoreOpener-338b47e35e9a8b9dd3045d365aaf47bd-1 {}] regionserver.HStore(327): Store=338b47e35e9a8b9dd3045d365aaf47bd/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T12:54:29,377 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for 338b47e35e9a8b9dd3045d365aaf47bd 2024-11-14T12:54:29,378 INFO [StoreOpener-333a3eba2bb2d318c041beb4642f5ef3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 333a3eba2bb2d318c041beb4642f5ef3 2024-11-14T12:54:29,379 INFO [StoreOpener-e90cc230c4f13fece446e3e8420dddbe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region e90cc230c4f13fece446e3e8420dddbe 2024-11-14T12:54:29,380 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/338b47e35e9a8b9dd3045d365aaf47bd 2024-11-14T12:54:29,381 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/338b47e35e9a8b9dd3045d365aaf47bd 2024-11-14T12:54:29,381 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure ea6e95809b33f853859fa227179bfe85, server=669ef94e8b26,46877,1731588865107 because future has completed 2024-11-14T12:54:29,382 INFO [StoreOpener-333a3eba2bb2d318c041beb4642f5ef3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 333a3eba2bb2d318c041beb4642f5ef3 columnFamilyName family 2024-11-14T12:54:29,382 DEBUG [StoreOpener-333a3eba2bb2d318c041beb4642f5ef3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T12:54:29,382 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for 338b47e35e9a8b9dd3045d365aaf47bd 2024-11-14T12:54:29,382 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for 338b47e35e9a8b9dd3045d365aaf47bd 2024-11-14T12:54:29,383 INFO [StoreOpener-e90cc230c4f13fece446e3e8420dddbe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e90cc230c4f13fece446e3e8420dddbe columnFamilyName family 2024-11-14T12:54:29,383 INFO [StoreOpener-333a3eba2bb2d318c041beb4642f5ef3-1 {}] regionserver.HStore(327): Store=333a3eba2bb2d318c041beb4642f5ef3/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T12:54:29,383 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for 333a3eba2bb2d318c041beb4642f5ef3 2024-11-14T12:54:29,383 DEBUG [StoreOpener-e90cc230c4f13fece446e3e8420dddbe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T12:54:29,384 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/333a3eba2bb2d318c041beb4642f5ef3 2024-11-14T12:54:29,384 INFO [StoreOpener-e90cc230c4f13fece446e3e8420dddbe-1 {}] regionserver.HStore(327): Store=e90cc230c4f13fece446e3e8420dddbe/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T12:54:29,385 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for e90cc230c4f13fece446e3e8420dddbe 2024-11-14T12:54:29,385 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/333a3eba2bb2d318c041beb4642f5ef3 2024-11-14T12:54:29,386 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for 333a3eba2bb2d318c041beb4642f5ef3 2024-11-14T12:54:29,386 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for 333a3eba2bb2d318c041beb4642f5ef3 2024-11-14T12:54:29,386 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/e90cc230c4f13fece446e3e8420dddbe 2024-11-14T12:54:29,386 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for 338b47e35e9a8b9dd3045d365aaf47bd 2024-11-14T12:54:29,387 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/e90cc230c4f13fece446e3e8420dddbe 2024-11-14T12:54:29,387 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for e90cc230c4f13fece446e3e8420dddbe 2024-11-14T12:54:29,387 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for e90cc230c4f13fece446e3e8420dddbe 2024-11-14T12:54:29,389 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for 333a3eba2bb2d318c041beb4642f5ef3 2024-11-14T12:54:29,390 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for e90cc230c4f13fece446e3e8420dddbe 2024-11-14T12:54:29,391 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/338b47e35e9a8b9dd3045d365aaf47bd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T12:54:29,392 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened 338b47e35e9a8b9dd3045d365aaf47bd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70062492, jitterRate=0.044012486934661865}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T12:54:29,392 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 338b47e35e9a8b9dd3045d365aaf47bd 2024-11-14T12:54:29,393 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for 338b47e35e9a8b9dd3045d365aaf47bd: Running coprocessor pre-open hook at 1731588869368Writing region info on filesystem at 1731588869368Initializing all the Stores at 1731588869370 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731588869370Cleaning up temporary data from old regions at 1731588869382 (+12 ms)Running coprocessor post-open hooks at 1731588869392 (+10 ms)Region opened successfully at 1731588869392 2024-11-14T12:54:29,393 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=20 2024-11-14T12:54:29,393 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure ea6e95809b33f853859fa227179bfe85, server=669ef94e8b26,46877,1731588865107 in 336 msec 2024-11-14T12:54:29,394 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1731588868017.338b47e35e9a8b9dd3045d365aaf47bd., pid=55, masterSystemTime=1731588869230 2024-11-14T12:54:29,397 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/e90cc230c4f13fece446e3e8420dddbe/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T12:54:29,399 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened e90cc230c4f13fece446e3e8420dddbe; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72968570, jitterRate=0.08731642365455627}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T12:54:29,399 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e90cc230c4f13fece446e3e8420dddbe 2024-11-14T12:54:29,399 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for e90cc230c4f13fece446e3e8420dddbe: Running coprocessor pre-open hook at 1731588869377Writing region info on filesystem at 1731588869377Initializing all the Stores at 1731588869378 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731588869378Cleaning up temporary data from old regions at 1731588869387 (+9 ms)Running coprocessor post-open hooks at 1731588869399 (+12 ms)Region opened successfully at 1731588869399 2024-11-14T12:54:29,399 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/333a3eba2bb2d318c041beb4642f5ef3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T12:54:29,400 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ooo,1731588868017.338b47e35e9a8b9dd3045d365aaf47bd. 2024-11-14T12:54:29,400 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened test,ooo,1731588868017.338b47e35e9a8b9dd3045d365aaf47bd. 2024-11-14T12:54:29,400 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(132): Open test,yyy,1731588868017.66b34b9be40614bd68d0cea995e44193. 2024-11-14T12:54:29,400 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7752): Opening region: {ENCODED => 66b34b9be40614bd68d0cea995e44193, NAME => 'test,yyy,1731588868017.66b34b9be40614bd68d0cea995e44193.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-14T12:54:29,400 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened 333a3eba2bb2d318c041beb4642f5ef3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68639079, jitterRate=0.02280198037624359}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T12:54:29,400 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 333a3eba2bb2d318c041beb4642f5ef3 2024-11-14T12:54:29,400 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for 333a3eba2bb2d318c041beb4642f5ef3: Running coprocessor pre-open hook at 1731588869375Writing region info on filesystem at 1731588869375Initializing all the Stores at 1731588869377 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731588869377Cleaning up temporary data from old regions at 1731588869386 (+9 ms)Running coprocessor post-open hooks at 1731588869400 (+14 ms)Region opened successfully at 1731588869400 2024-11-14T12:54:29,400 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 66b34b9be40614bd68d0cea995e44193 2024-11-14T12:54:29,401 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(898): Instantiated test,yyy,1731588868017.66b34b9be40614bd68d0cea995e44193.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:29,401 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7794): checking encryption for 66b34b9be40614bd68d0cea995e44193 2024-11-14T12:54:29,401 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7797): checking classloading for 66b34b9be40614bd68d0cea995e44193 2024-11-14T12:54:29,401 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1731588868017.e90cc230c4f13fece446e3e8420dddbe., pid=38, masterSystemTime=1731588869140 2024-11-14T12:54:29,402 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=ea6e95809b33f853859fa227179bfe85, ASSIGN in 584 msec 2024-11-14T12:54:29,403 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1731588868017.333a3eba2bb2d318c041beb4642f5ef3., pid=47, masterSystemTime=1731588869168 2024-11-14T12:54:29,405 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=338b47e35e9a8b9dd3045d365aaf47bd, regionState=OPEN, openSeqNum=2, regionLocation=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:29,405 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1731588868017.e90cc230c4f13fece446e3e8420dddbe. 2024-11-14T12:54:29,405 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened test,mmm,1731588868017.e90cc230c4f13fece446e3e8420dddbe. 2024-11-14T12:54:29,406 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open test,xxx,1731588868017.cadcf21958eaec53ee489ca0918e3ab7. 2024-11-14T12:54:29,406 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => cadcf21958eaec53ee489ca0918e3ab7, NAME => 'test,xxx,1731588868017.cadcf21958eaec53ee489ca0918e3ab7.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-11-14T12:54:29,406 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test cadcf21958eaec53ee489ca0918e3ab7 2024-11-14T12:54:29,406 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated test,xxx,1731588868017.cadcf21958eaec53ee489ca0918e3ab7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:29,407 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for cadcf21958eaec53ee489ca0918e3ab7 2024-11-14T12:54:29,407 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for cadcf21958eaec53ee489ca0918e3ab7 2024-11-14T12:54:29,408 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for test,kkk,1731588868017.333a3eba2bb2d318c041beb4642f5ef3. 2024-11-14T12:54:29,408 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened test,kkk,1731588868017.333a3eba2bb2d318c041beb4642f5ef3. 2024-11-14T12:54:29,408 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 338b47e35e9a8b9dd3045d365aaf47bd, server=669ef94e8b26,43475,1731588864980 because future has completed 2024-11-14T12:54:29,408 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open test,eee,1731588868017.23f7fbc112f1353900c1181e3ed36707. 2024-11-14T12:54:29,409 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => 23f7fbc112f1353900c1181e3ed36707, NAME => 'test,eee,1731588868017.23f7fbc112f1353900c1181e3ed36707.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-14T12:54:29,409 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 23f7fbc112f1353900c1181e3ed36707 2024-11-14T12:54:29,409 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated test,eee,1731588868017.23f7fbc112f1353900c1181e3ed36707.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:29,410 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for 23f7fbc112f1353900c1181e3ed36707 2024-11-14T12:54:29,410 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=e90cc230c4f13fece446e3e8420dddbe, regionState=OPEN, openSeqNum=2, regionLocation=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:29,410 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for 23f7fbc112f1353900c1181e3ed36707 2024-11-14T12:54:29,412 INFO [StoreOpener-66b34b9be40614bd68d0cea995e44193-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 66b34b9be40614bd68d0cea995e44193 2024-11-14T12:54:29,413 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure e90cc230c4f13fece446e3e8420dddbe, server=669ef94e8b26,46877,1731588865107 because future has completed 2024-11-14T12:54:29,414 INFO [StoreOpener-66b34b9be40614bd68d0cea995e44193-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 66b34b9be40614bd68d0cea995e44193 columnFamilyName family 2024-11-14T12:54:29,415 DEBUG [StoreOpener-66b34b9be40614bd68d0cea995e44193-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T12:54:29,415 INFO [StoreOpener-66b34b9be40614bd68d0cea995e44193-1 {}] regionserver.HStore(327): Store=66b34b9be40614bd68d0cea995e44193/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T12:54:29,416 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1038): replaying wal for 66b34b9be40614bd68d0cea995e44193 2024-11-14T12:54:29,415 INFO [StoreOpener-cadcf21958eaec53ee489ca0918e3ab7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region cadcf21958eaec53ee489ca0918e3ab7 2024-11-14T12:54:29,417 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/66b34b9be40614bd68d0cea995e44193 2024-11-14T12:54:29,419 INFO [StoreOpener-23f7fbc112f1353900c1181e3ed36707-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 23f7fbc112f1353900c1181e3ed36707 2024-11-14T12:54:29,419 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/66b34b9be40614bd68d0cea995e44193 2024-11-14T12:54:29,420 INFO [StoreOpener-cadcf21958eaec53ee489ca0918e3ab7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cadcf21958eaec53ee489ca0918e3ab7 columnFamilyName family 2024-11-14T12:54:29,420 DEBUG [StoreOpener-cadcf21958eaec53ee489ca0918e3ab7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T12:54:29,420 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1048): stopping wal replay for 66b34b9be40614bd68d0cea995e44193 2024-11-14T12:54:29,420 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1060): Cleaning up temporary data for 66b34b9be40614bd68d0cea995e44193 2024-11-14T12:54:29,421 INFO [StoreOpener-cadcf21958eaec53ee489ca0918e3ab7-1 {}] regionserver.HStore(327): Store=cadcf21958eaec53ee489ca0918e3ab7/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T12:54:29,421 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for cadcf21958eaec53ee489ca0918e3ab7 2024-11-14T12:54:29,422 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/cadcf21958eaec53ee489ca0918e3ab7 2024-11-14T12:54:29,422 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=19 2024-11-14T12:54:29,423 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=19, state=SUCCESS, hasLock=false; OpenRegionProcedure 338b47e35e9a8b9dd3045d365aaf47bd, server=669ef94e8b26,43475,1731588864980 in 260 msec 2024-11-14T12:54:29,423 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/cadcf21958eaec53ee489ca0918e3ab7 2024-11-14T12:54:29,423 INFO [StoreOpener-23f7fbc112f1353900c1181e3ed36707-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 23f7fbc112f1353900c1181e3ed36707 columnFamilyName family 2024-11-14T12:54:29,423 DEBUG [StoreOpener-23f7fbc112f1353900c1181e3ed36707-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T12:54:29,423 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for cadcf21958eaec53ee489ca0918e3ab7 2024-11-14T12:54:29,423 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1093): writing seq id for 66b34b9be40614bd68d0cea995e44193 2024-11-14T12:54:29,423 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for cadcf21958eaec53ee489ca0918e3ab7 2024-11-14T12:54:29,424 INFO [StoreOpener-23f7fbc112f1353900c1181e3ed36707-1 {}] regionserver.HStore(327): Store=23f7fbc112f1353900c1181e3ed36707/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T12:54:29,424 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for 23f7fbc112f1353900c1181e3ed36707 2024-11-14T12:54:29,425 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/23f7fbc112f1353900c1181e3ed36707 2024-11-14T12:54:29,425 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/23f7fbc112f1353900c1181e3ed36707 2024-11-14T12:54:29,425 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for cadcf21958eaec53ee489ca0918e3ab7 2024-11-14T12:54:29,426 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for 23f7fbc112f1353900c1181e3ed36707 2024-11-14T12:54:29,426 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for 23f7fbc112f1353900c1181e3ed36707 2024-11-14T12:54:29,428 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/66b34b9be40614bd68d0cea995e44193/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T12:54:29,430 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for 23f7fbc112f1353900c1181e3ed36707 2024-11-14T12:54:29,429 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1114): Opened 66b34b9be40614bd68d0cea995e44193; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65702411, jitterRate=-0.020957782864570618}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T12:54:29,430 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/cadcf21958eaec53ee489ca0918e3ab7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T12:54:29,430 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 66b34b9be40614bd68d0cea995e44193 2024-11-14T12:54:29,430 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1006): Region open journal for 66b34b9be40614bd68d0cea995e44193: Running coprocessor pre-open hook at 1731588869401Writing region info on filesystem at 1731588869401Initializing all the Stores at 1731588869404 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731588869404Cleaning up temporary data from old regions at 1731588869420 (+16 ms)Running coprocessor post-open hooks at 1731588869430 (+10 ms)Region opened successfully at 1731588869430 2024-11-14T12:54:29,430 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened cadcf21958eaec53ee489ca0918e3ab7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66508255, jitterRate=-0.008949771523475647}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T12:54:29,431 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cadcf21958eaec53ee489ca0918e3ab7 2024-11-14T12:54:29,431 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for cadcf21958eaec53ee489ca0918e3ab7: Running coprocessor pre-open hook at 1731588869407Writing region info on filesystem at 1731588869407Initializing all the Stores at 1731588869409 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731588869409Cleaning up temporary data from old regions at 1731588869423 (+14 ms)Running coprocessor post-open hooks at 1731588869431 (+8 ms)Region opened successfully at 1731588869431 2024-11-14T12:54:29,432 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=338b47e35e9a8b9dd3045d365aaf47bd, ASSIGN in 614 msec 2024-11-14T12:54:29,433 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1731588868017.66b34b9be40614bd68d0cea995e44193., pid=44, masterSystemTime=1731588869230 2024-11-14T12:54:29,433 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1731588868017.cadcf21958eaec53ee489ca0918e3ab7., pid=31, masterSystemTime=1731588869140 2024-11-14T12:54:29,438 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1731588868017.cadcf21958eaec53ee489ca0918e3ab7. 2024-11-14T12:54:29,438 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened test,xxx,1731588868017.cadcf21958eaec53ee489ca0918e3ab7. 2024-11-14T12:54:29,439 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open test,vvv,1731588868017.87eb074dbd46b3bc6e27fb0bf3291d5b. 2024-11-14T12:54:29,439 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => 87eb074dbd46b3bc6e27fb0bf3291d5b, NAME => 'test,vvv,1731588868017.87eb074dbd46b3bc6e27fb0bf3291d5b.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-14T12:54:29,439 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 87eb074dbd46b3bc6e27fb0bf3291d5b 2024-11-14T12:54:29,440 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated test,vvv,1731588868017.87eb074dbd46b3bc6e27fb0bf3291d5b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:29,440 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for 87eb074dbd46b3bc6e27fb0bf3291d5b 2024-11-14T12:54:29,440 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for 87eb074dbd46b3bc6e27fb0bf3291d5b 2024-11-14T12:54:29,441 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1731588868017.66b34b9be40614bd68d0cea995e44193. 2024-11-14T12:54:29,441 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(153): Opened test,yyy,1731588868017.66b34b9be40614bd68d0cea995e44193. 2024-11-14T12:54:29,441 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/23f7fbc112f1353900c1181e3ed36707/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T12:54:29,441 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(132): Open test,bbb,1731588868017.c2ccfc8c17ab1571bc4ea6d526017f9f. 2024-11-14T12:54:29,442 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7752): Opening region: {ENCODED => c2ccfc8c17ab1571bc4ea6d526017f9f, NAME => 'test,bbb,1731588868017.c2ccfc8c17ab1571bc4ea6d526017f9f.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-11-14T12:54:29,442 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c2ccfc8c17ab1571bc4ea6d526017f9f 2024-11-14T12:54:29,442 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(898): Instantiated test,bbb,1731588868017.c2ccfc8c17ab1571bc4ea6d526017f9f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:29,442 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7794): checking encryption for c2ccfc8c17ab1571bc4ea6d526017f9f 2024-11-14T12:54:29,442 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7797): checking classloading for c2ccfc8c17ab1571bc4ea6d526017f9f 2024-11-14T12:54:29,442 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened 23f7fbc112f1353900c1181e3ed36707; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65033699, jitterRate=-0.03092236816883087}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T12:54:29,443 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 23f7fbc112f1353900c1181e3ed36707 2024-11-14T12:54:29,443 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for 23f7fbc112f1353900c1181e3ed36707: Running coprocessor pre-open hook at 1731588869410Writing region info on filesystem at 1731588869410Initializing all the Stores at 1731588869411 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731588869412 (+1 ms)Cleaning up temporary data from old regions at 1731588869426 (+14 ms)Running coprocessor post-open hooks at 1731588869443 (+17 ms)Region opened successfully at 1731588869443 2024-11-14T12:54:29,448 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1731588868017.23f7fbc112f1353900c1181e3ed36707., pid=46, masterSystemTime=1731588869168 2024-11-14T12:54:29,448 INFO [StoreOpener-87eb074dbd46b3bc6e27fb0bf3291d5b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 87eb074dbd46b3bc6e27fb0bf3291d5b 2024-11-14T12:54:29,450 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=17 2024-11-14T12:54:29,450 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure e90cc230c4f13fece446e3e8420dddbe, server=669ef94e8b26,46877,1731588865107 in 392 msec 2024-11-14T12:54:29,451 INFO [StoreOpener-87eb074dbd46b3bc6e27fb0bf3291d5b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 87eb074dbd46b3bc6e27fb0bf3291d5b columnFamilyName family 2024-11-14T12:54:29,451 DEBUG [StoreOpener-87eb074dbd46b3bc6e27fb0bf3291d5b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T12:54:29,451 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1731588868017.23f7fbc112f1353900c1181e3ed36707. 2024-11-14T12:54:29,451 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened test,eee,1731588868017.23f7fbc112f1353900c1181e3ed36707. 2024-11-14T12:54:29,451 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open test,www,1731588868017.22b9f10374fc1ea7a876d476ce0b083d. 2024-11-14T12:54:29,452 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => 22b9f10374fc1ea7a876d476ce0b083d, NAME => 'test,www,1731588868017.22b9f10374fc1ea7a876d476ce0b083d.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-11-14T12:54:29,452 INFO [StoreOpener-87eb074dbd46b3bc6e27fb0bf3291d5b-1 {}] regionserver.HStore(327): Store=87eb074dbd46b3bc6e27fb0bf3291d5b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T12:54:29,452 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 22b9f10374fc1ea7a876d476ce0b083d 2024-11-14T12:54:29,452 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for 87eb074dbd46b3bc6e27fb0bf3291d5b 2024-11-14T12:54:29,452 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated test,www,1731588868017.22b9f10374fc1ea7a876d476ce0b083d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:29,452 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for 22b9f10374fc1ea7a876d476ce0b083d 2024-11-14T12:54:29,452 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for 22b9f10374fc1ea7a876d476ce0b083d 2024-11-14T12:54:29,453 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/87eb074dbd46b3bc6e27fb0bf3291d5b 2024-11-14T12:54:29,454 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e90cc230c4f13fece446e3e8420dddbe, ASSIGN in 641 msec 2024-11-14T12:54:29,454 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/87eb074dbd46b3bc6e27fb0bf3291d5b 2024-11-14T12:54:29,455 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for 87eb074dbd46b3bc6e27fb0bf3291d5b 2024-11-14T12:54:29,455 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for 87eb074dbd46b3bc6e27fb0bf3291d5b 2024-11-14T12:54:29,456 INFO [StoreOpener-22b9f10374fc1ea7a876d476ce0b083d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 22b9f10374fc1ea7a876d476ce0b083d 2024-11-14T12:54:29,456 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=23f7fbc112f1353900c1181e3ed36707, regionState=OPEN, openSeqNum=2, regionLocation=669ef94e8b26,44819,1731588865173 2024-11-14T12:54:29,457 INFO [StoreOpener-c2ccfc8c17ab1571bc4ea6d526017f9f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c2ccfc8c17ab1571bc4ea6d526017f9f 2024-11-14T12:54:29,458 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for 87eb074dbd46b3bc6e27fb0bf3291d5b 2024-11-14T12:54:29,458 INFO [StoreOpener-22b9f10374fc1ea7a876d476ce0b083d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 22b9f10374fc1ea7a876d476ce0b083d columnFamilyName family 2024-11-14T12:54:29,459 DEBUG [StoreOpener-22b9f10374fc1ea7a876d476ce0b083d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T12:54:29,461 INFO [StoreOpener-22b9f10374fc1ea7a876d476ce0b083d-1 {}] regionserver.HStore(327): Store=22b9f10374fc1ea7a876d476ce0b083d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T12:54:29,461 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for 22b9f10374fc1ea7a876d476ce0b083d 2024-11-14T12:54:29,461 INFO [StoreOpener-c2ccfc8c17ab1571bc4ea6d526017f9f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c2ccfc8c17ab1571bc4ea6d526017f9f columnFamilyName family 2024-11-14T12:54:29,461 DEBUG [StoreOpener-c2ccfc8c17ab1571bc4ea6d526017f9f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T12:54:29,461 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 23f7fbc112f1353900c1181e3ed36707, server=669ef94e8b26,44819,1731588865173 because future has completed 2024-11-14T12:54:29,462 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/22b9f10374fc1ea7a876d476ce0b083d 2024-11-14T12:54:29,462 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=66b34b9be40614bd68d0cea995e44193, regionState=OPEN, openSeqNum=2, regionLocation=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:29,463 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/22b9f10374fc1ea7a876d476ce0b083d 2024-11-14T12:54:29,463 INFO [StoreOpener-c2ccfc8c17ab1571bc4ea6d526017f9f-1 {}] regionserver.HStore(327): Store=c2ccfc8c17ab1571bc4ea6d526017f9f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T12:54:29,463 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1038): replaying wal for c2ccfc8c17ab1571bc4ea6d526017f9f 2024-11-14T12:54:29,464 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for 22b9f10374fc1ea7a876d476ce0b083d 2024-11-14T12:54:29,464 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for 22b9f10374fc1ea7a876d476ce0b083d 2024-11-14T12:54:29,464 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/c2ccfc8c17ab1571bc4ea6d526017f9f 2024-11-14T12:54:29,465 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/c2ccfc8c17ab1571bc4ea6d526017f9f 2024-11-14T12:54:29,466 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1048): stopping wal replay for c2ccfc8c17ab1571bc4ea6d526017f9f 2024-11-14T12:54:29,466 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1060): Cleaning up temporary data for c2ccfc8c17ab1571bc4ea6d526017f9f 2024-11-14T12:54:29,469 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66b34b9be40614bd68d0cea995e44193, server=669ef94e8b26,43475,1731588864980 because future has completed 2024-11-14T12:54:29,470 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for 22b9f10374fc1ea7a876d476ce0b083d 2024-11-14T12:54:29,470 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1093): writing seq id for c2ccfc8c17ab1571bc4ea6d526017f9f 2024-11-14T12:54:29,475 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=9 2024-11-14T12:54:29,475 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure 23f7fbc112f1353900c1181e3ed36707, server=669ef94e8b26,44819,1731588865173 in 375 msec 2024-11-14T12:54:29,479 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/87eb074dbd46b3bc6e27fb0bf3291d5b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T12:54:29,481 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/22b9f10374fc1ea7a876d476ce0b083d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T12:54:29,482 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened 22b9f10374fc1ea7a876d476ce0b083d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65962675, jitterRate=-0.01707954704761505}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T12:54:29,482 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 22b9f10374fc1ea7a876d476ce0b083d 2024-11-14T12:54:29,482 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/c2ccfc8c17ab1571bc4ea6d526017f9f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T12:54:29,482 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened 87eb074dbd46b3bc6e27fb0bf3291d5b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74849256, jitterRate=0.11534082889556885}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T12:54:29,482 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for 22b9f10374fc1ea7a876d476ce0b083d: Running coprocessor pre-open hook at 1731588869452Writing region info on filesystem at 1731588869452Initializing all the Stores at 1731588869455 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731588869455Cleaning up temporary data from old regions at 1731588869464 (+9 ms)Running coprocessor post-open hooks at 1731588869482 (+18 ms)Region opened successfully at 1731588869482 2024-11-14T12:54:29,482 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 87eb074dbd46b3bc6e27fb0bf3291d5b 2024-11-14T12:54:29,482 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for 87eb074dbd46b3bc6e27fb0bf3291d5b: Running coprocessor pre-open hook at 1731588869440Writing region info on filesystem at 1731588869440Initializing all the Stores at 1731588869442 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731588869442Cleaning up temporary data from old regions at 1731588869455 (+13 ms)Running coprocessor post-open hooks at 1731588869482 (+27 ms)Region opened successfully at 1731588869482 2024-11-14T12:54:29,483 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1114): Opened c2ccfc8c17ab1571bc4ea6d526017f9f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66356033, jitterRate=-0.011218056082725525}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T12:54:29,483 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c2ccfc8c17ab1571bc4ea6d526017f9f 2024-11-14T12:54:29,484 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1006): Region open journal for c2ccfc8c17ab1571bc4ea6d526017f9f: Running coprocessor pre-open hook at 1731588869442Writing region info on filesystem at 1731588869442Initializing all the Stores at 1731588869449 (+7 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731588869449Cleaning up temporary data from old regions at 1731588869466 (+17 ms)Running coprocessor post-open hooks at 1731588869483 (+17 ms)Region opened successfully at 1731588869483 2024-11-14T12:54:29,484 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for test,www,1731588868017.22b9f10374fc1ea7a876d476ce0b083d., pid=34, masterSystemTime=1731588869168 2024-11-14T12:54:29,484 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1731588868017.87eb074dbd46b3bc6e27fb0bf3291d5b., pid=40, masterSystemTime=1731588869140 2024-11-14T12:54:29,485 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2236): Post open deploy tasks for test,bbb,1731588868017.c2ccfc8c17ab1571bc4ea6d526017f9f., pid=56, masterSystemTime=1731588869230 2024-11-14T12:54:29,485 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=23f7fbc112f1353900c1181e3ed36707, ASSIGN in 666 msec 2024-11-14T12:54:29,487 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for test,www,1731588868017.22b9f10374fc1ea7a876d476ce0b083d. 2024-11-14T12:54:29,487 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened test,www,1731588868017.22b9f10374fc1ea7a876d476ce0b083d. 2024-11-14T12:54:29,488 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open test,,1731588868017.a572ae08c8738aa198c40f32aa4b5f96. 2024-11-14T12:54:29,488 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => a572ae08c8738aa198c40f32aa4b5f96, NAME => 'test,,1731588868017.a572ae08c8738aa198c40f32aa4b5f96.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-14T12:54:29,489 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1731588868017.87eb074dbd46b3bc6e27fb0bf3291d5b. 2024-11-14T12:54:29,489 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened test,vvv,1731588868017.87eb074dbd46b3bc6e27fb0bf3291d5b. 2024-11-14T12:54:29,490 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test a572ae08c8738aa198c40f32aa4b5f96 2024-11-14T12:54:29,490 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated test,,1731588868017.a572ae08c8738aa198c40f32aa4b5f96.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:29,490 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for a572ae08c8738aa198c40f32aa4b5f96 2024-11-14T12:54:29,490 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for a572ae08c8738aa198c40f32aa4b5f96 2024-11-14T12:54:29,490 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2266): Finished post open deploy task for test,bbb,1731588868017.c2ccfc8c17ab1571bc4ea6d526017f9f. 2024-11-14T12:54:29,490 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(153): Opened test,bbb,1731588868017.c2ccfc8c17ab1571bc4ea6d526017f9f. 2024-11-14T12:54:29,490 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(132): Open test,rrr,1731588868017.a5d884ba0737bc572475e022a6ed1bc8. 2024-11-14T12:54:29,491 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7752): Opening region: {ENCODED => a5d884ba0737bc572475e022a6ed1bc8, NAME => 'test,rrr,1731588868017.a5d884ba0737bc572475e022a6ed1bc8.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-14T12:54:29,491 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test a5d884ba0737bc572475e022a6ed1bc8 2024-11-14T12:54:29,491 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(898): Instantiated test,rrr,1731588868017.a5d884ba0737bc572475e022a6ed1bc8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:29,491 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=29 2024-11-14T12:54:29,491 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7794): checking encryption for a5d884ba0737bc572475e022a6ed1bc8 2024-11-14T12:54:29,491 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7797): checking classloading for a5d884ba0737bc572475e022a6ed1bc8 2024-11-14T12:54:29,491 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 66b34b9be40614bd68d0cea995e44193, server=669ef94e8b26,43475,1731588864980 in 403 msec 2024-11-14T12:54:29,494 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=66b34b9be40614bd68d0cea995e44193, ASSIGN in 683 msec 2024-11-14T12:54:29,495 INFO [StoreOpener-a572ae08c8738aa198c40f32aa4b5f96-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region a572ae08c8738aa198c40f32aa4b5f96 2024-11-14T12:54:29,496 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=c2ccfc8c17ab1571bc4ea6d526017f9f, regionState=OPEN, openSeqNum=2, regionLocation=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:29,498 INFO [StoreOpener-a5d884ba0737bc572475e022a6ed1bc8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region a5d884ba0737bc572475e022a6ed1bc8 2024-11-14T12:54:29,500 INFO [StoreOpener-a572ae08c8738aa198c40f32aa4b5f96-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a572ae08c8738aa198c40f32aa4b5f96 columnFamilyName family 2024-11-14T12:54:29,500 DEBUG [StoreOpener-a572ae08c8738aa198c40f32aa4b5f96-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T12:54:29,501 INFO [StoreOpener-a572ae08c8738aa198c40f32aa4b5f96-1 {}] regionserver.HStore(327): Store=a572ae08c8738aa198c40f32aa4b5f96/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T12:54:29,501 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for a572ae08c8738aa198c40f32aa4b5f96 2024-11-14T12:54:29,502 INFO [StoreOpener-a5d884ba0737bc572475e022a6ed1bc8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a5d884ba0737bc572475e022a6ed1bc8 columnFamilyName family 2024-11-14T12:54:29,503 DEBUG [StoreOpener-a5d884ba0737bc572475e022a6ed1bc8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T12:54:29,505 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/a572ae08c8738aa198c40f32aa4b5f96 2024-11-14T12:54:29,505 INFO [StoreOpener-a5d884ba0737bc572475e022a6ed1bc8-1 {}] regionserver.HStore(327): Store=a5d884ba0737bc572475e022a6ed1bc8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T12:54:29,506 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1038): replaying wal for a5d884ba0737bc572475e022a6ed1bc8 2024-11-14T12:54:29,506 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/a572ae08c8738aa198c40f32aa4b5f96 2024-11-14T12:54:29,507 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for a572ae08c8738aa198c40f32aa4b5f96 2024-11-14T12:54:29,507 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for a572ae08c8738aa198c40f32aa4b5f96 2024-11-14T12:54:29,507 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure c2ccfc8c17ab1571bc4ea6d526017f9f, server=669ef94e8b26,43475,1731588864980 because future has completed 2024-11-14T12:54:29,508 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/a5d884ba0737bc572475e022a6ed1bc8 2024-11-14T12:54:29,508 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=87eb074dbd46b3bc6e27fb0bf3291d5b, regionState=OPEN, openSeqNum=2, regionLocation=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:29,510 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/a5d884ba0737bc572475e022a6ed1bc8 2024-11-14T12:54:29,511 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1048): stopping wal replay for a5d884ba0737bc572475e022a6ed1bc8 2024-11-14T12:54:29,511 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1060): Cleaning up temporary data for a5d884ba0737bc572475e022a6ed1bc8 2024-11-14T12:54:29,512 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for a572ae08c8738aa198c40f32aa4b5f96 2024-11-14T12:54:29,513 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 87eb074dbd46b3bc6e27fb0bf3291d5b, server=669ef94e8b26,46877,1731588865107 because future has completed 2024-11-14T12:54:29,514 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): writing seq id for a5d884ba0737bc572475e022a6ed1bc8 2024-11-14T12:54:29,522 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=6 2024-11-14T12:54:29,522 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure c2ccfc8c17ab1571bc4ea6d526017f9f, server=669ef94e8b26,43475,1731588864980 in 343 msec 2024-11-14T12:54:29,526 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/a572ae08c8738aa198c40f32aa4b5f96/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T12:54:29,527 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=c2ccfc8c17ab1571bc4ea6d526017f9f, ASSIGN in 713 msec 2024-11-14T12:54:29,528 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened a572ae08c8738aa198c40f32aa4b5f96; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70054534, jitterRate=0.043893903493881226}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T12:54:29,528 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a572ae08c8738aa198c40f32aa4b5f96 2024-11-14T12:54:29,529 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for a572ae08c8738aa198c40f32aa4b5f96: Running coprocessor pre-open hook at 1731588869490Writing region info on filesystem at 1731588869490Initializing all the Stores at 1731588869492 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731588869492Cleaning up temporary data from old regions at 1731588869507 (+15 ms)Running coprocessor post-open hooks at 1731588869529 (+22 ms)Region opened successfully at 1731588869529 2024-11-14T12:54:29,531 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1731588868017.a572ae08c8738aa198c40f32aa4b5f96., pid=45, masterSystemTime=1731588869168 2024-11-14T12:54:29,536 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=26 2024-11-14T12:54:29,536 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 87eb074dbd46b3bc6e27fb0bf3291d5b, server=669ef94e8b26,46877,1731588865107 in 476 msec 2024-11-14T12:54:29,537 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for test,,1731588868017.a572ae08c8738aa198c40f32aa4b5f96. 2024-11-14T12:54:29,537 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened test,,1731588868017.a572ae08c8738aa198c40f32aa4b5f96. 2024-11-14T12:54:29,537 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open test,ttt,1731588868017.c8ddc4f331afb3a3007ef5c49cdf6ab8. 2024-11-14T12:54:29,537 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => c8ddc4f331afb3a3007ef5c49cdf6ab8, NAME => 'test,ttt,1731588868017.c8ddc4f331afb3a3007ef5c49cdf6ab8.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-14T12:54:29,538 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c8ddc4f331afb3a3007ef5c49cdf6ab8 2024-11-14T12:54:29,538 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated test,ttt,1731588868017.c8ddc4f331afb3a3007ef5c49cdf6ab8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:29,538 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for c8ddc4f331afb3a3007ef5c49cdf6ab8 2024-11-14T12:54:29,538 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for c8ddc4f331afb3a3007ef5c49cdf6ab8 2024-11-14T12:54:29,539 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/a5d884ba0737bc572475e022a6ed1bc8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T12:54:29,539 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=87eb074dbd46b3bc6e27fb0bf3291d5b, ASSIGN in 728 msec 2024-11-14T12:54:29,540 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1114): Opened a5d884ba0737bc572475e022a6ed1bc8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69897987, jitterRate=0.04156117141246796}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T12:54:29,540 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a5d884ba0737bc572475e022a6ed1bc8 2024-11-14T12:54:29,540 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1006): Region open journal for a5d884ba0737bc572475e022a6ed1bc8: Running coprocessor pre-open hook at 1731588869492Writing region info on filesystem at 1731588869492Initializing all the Stores at 1731588869493 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731588869493Cleaning up temporary data from old regions at 1731588869511 (+18 ms)Running coprocessor post-open hooks at 1731588869540 (+29 ms)Region opened successfully at 1731588869540 2024-11-14T12:54:29,541 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1731588868017.a5d884ba0737bc572475e022a6ed1bc8., pid=50, masterSystemTime=1731588869230 2024-11-14T12:54:29,541 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=a572ae08c8738aa198c40f32aa4b5f96, regionState=OPEN, openSeqNum=2, regionLocation=669ef94e8b26,44819,1731588865173 2024-11-14T12:54:29,544 INFO [StoreOpener-c8ddc4f331afb3a3007ef5c49cdf6ab8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c8ddc4f331afb3a3007ef5c49cdf6ab8 2024-11-14T12:54:29,546 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure a572ae08c8738aa198c40f32aa4b5f96, server=669ef94e8b26,44819,1731588865173 because future has completed 2024-11-14T12:54:29,546 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1731588868017.a5d884ba0737bc572475e022a6ed1bc8. 2024-11-14T12:54:29,546 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(153): Opened test,rrr,1731588868017.a5d884ba0737bc572475e022a6ed1bc8. 2024-11-14T12:54:29,547 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open test,uuu,1731588868017.f8c0d62856832eec9f9ca20bb6db602c. 2024-11-14T12:54:29,547 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => f8c0d62856832eec9f9ca20bb6db602c, NAME => 'test,uuu,1731588868017.f8c0d62856832eec9f9ca20bb6db602c.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-14T12:54:29,548 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f8c0d62856832eec9f9ca20bb6db602c 2024-11-14T12:54:29,548 INFO [StoreOpener-c8ddc4f331afb3a3007ef5c49cdf6ab8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c8ddc4f331afb3a3007ef5c49cdf6ab8 columnFamilyName family 2024-11-14T12:54:29,548 DEBUG [StoreOpener-c8ddc4f331afb3a3007ef5c49cdf6ab8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T12:54:29,548 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated test,uuu,1731588868017.f8c0d62856832eec9f9ca20bb6db602c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:29,548 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for f8c0d62856832eec9f9ca20bb6db602c 2024-11-14T12:54:29,548 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for f8c0d62856832eec9f9ca20bb6db602c 2024-11-14T12:54:29,549 INFO [StoreOpener-c8ddc4f331afb3a3007ef5c49cdf6ab8-1 {}] regionserver.HStore(327): Store=c8ddc4f331afb3a3007ef5c49cdf6ab8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T12:54:29,549 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for c8ddc4f331afb3a3007ef5c49cdf6ab8 2024-11-14T12:54:29,550 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/c8ddc4f331afb3a3007ef5c49cdf6ab8 2024-11-14T12:54:29,551 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/c8ddc4f331afb3a3007ef5c49cdf6ab8 2024-11-14T12:54:29,551 INFO [StoreOpener-f8c0d62856832eec9f9ca20bb6db602c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f8c0d62856832eec9f9ca20bb6db602c 2024-11-14T12:54:29,551 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for c8ddc4f331afb3a3007ef5c49cdf6ab8 2024-11-14T12:54:29,551 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for c8ddc4f331afb3a3007ef5c49cdf6ab8 2024-11-14T12:54:29,553 INFO [StoreOpener-f8c0d62856832eec9f9ca20bb6db602c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f8c0d62856832eec9f9ca20bb6db602c columnFamilyName family 2024-11-14T12:54:29,553 DEBUG [StoreOpener-f8c0d62856832eec9f9ca20bb6db602c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T12:54:29,554 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=5 2024-11-14T12:54:29,554 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure a572ae08c8738aa198c40f32aa4b5f96, server=669ef94e8b26,44819,1731588865173 in 461 msec 2024-11-14T12:54:29,554 INFO [StoreOpener-f8c0d62856832eec9f9ca20bb6db602c-1 {}] regionserver.HStore(327): Store=f8c0d62856832eec9f9ca20bb6db602c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T12:54:29,554 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for f8c0d62856832eec9f9ca20bb6db602c 2024-11-14T12:54:29,558 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN in 745 msec 2024-11-14T12:54:29,560 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/f8c0d62856832eec9f9ca20bb6db602c 2024-11-14T12:54:29,561 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for c8ddc4f331afb3a3007ef5c49cdf6ab8 2024-11-14T12:54:29,561 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/f8c0d62856832eec9f9ca20bb6db602c 2024-11-14T12:54:29,561 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for f8c0d62856832eec9f9ca20bb6db602c 2024-11-14T12:54:29,561 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for f8c0d62856832eec9f9ca20bb6db602c 2024-11-14T12:54:29,562 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=a5d884ba0737bc572475e022a6ed1bc8, regionState=OPEN, openSeqNum=2, regionLocation=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:29,566 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for f8c0d62856832eec9f9ca20bb6db602c 2024-11-14T12:54:29,566 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure a5d884ba0737bc572475e022a6ed1bc8, server=669ef94e8b26,43475,1731588864980 because future has completed 2024-11-14T12:54:29,567 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/c8ddc4f331afb3a3007ef5c49cdf6ab8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T12:54:29,568 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=22b9f10374fc1ea7a876d476ce0b083d, regionState=OPEN, openSeqNum=2, regionLocation=669ef94e8b26,44819,1731588865173 2024-11-14T12:54:29,568 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened c8ddc4f331afb3a3007ef5c49cdf6ab8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67867606, jitterRate=0.011306136846542358}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T12:54:29,568 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c8ddc4f331afb3a3007ef5c49cdf6ab8 2024-11-14T12:54:29,568 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for c8ddc4f331afb3a3007ef5c49cdf6ab8: Running coprocessor pre-open hook at 1731588869538Writing region info on filesystem at 1731588869538Initializing all the Stores at 1731588869540 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731588869540Cleaning up temporary data from old regions at 1731588869551 (+11 ms)Running coprocessor post-open hooks at 1731588869568 (+17 ms)Region opened successfully at 1731588869568 2024-11-14T12:54:29,570 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1731588868017.c8ddc4f331afb3a3007ef5c49cdf6ab8., pid=41, masterSystemTime=1731588869168 2024-11-14T12:54:29,570 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/f8c0d62856832eec9f9ca20bb6db602c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T12:54:29,570 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened f8c0d62856832eec9f9ca20bb6db602c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68091764, jitterRate=0.014646351337432861}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T12:54:29,570 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f8c0d62856832eec9f9ca20bb6db602c 2024-11-14T12:54:29,571 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for f8c0d62856832eec9f9ca20bb6db602c: Running coprocessor pre-open hook at 1731588869548Writing region info on filesystem at 1731588869548Initializing all the Stores at 1731588869550 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731588869550Cleaning up temporary data from old regions at 1731588869561 (+11 ms)Running coprocessor post-open hooks at 1731588869570 (+9 ms)Region opened successfully at 1731588869570 2024-11-14T12:54:29,572 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1731588868017.f8c0d62856832eec9f9ca20bb6db602c., pid=43, masterSystemTime=1731588869230 2024-11-14T12:54:29,574 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1731588868017.c8ddc4f331afb3a3007ef5c49cdf6ab8. 2024-11-14T12:54:29,574 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened test,ttt,1731588868017.c8ddc4f331afb3a3007ef5c49cdf6ab8. 2024-11-14T12:54:29,575 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 22b9f10374fc1ea7a876d476ce0b083d, server=669ef94e8b26,44819,1731588865173 because future has completed 2024-11-14T12:54:29,575 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for test,uuu,1731588868017.f8c0d62856832eec9f9ca20bb6db602c. 2024-11-14T12:54:29,576 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened test,uuu,1731588868017.f8c0d62856832eec9f9ca20bb6db602c. 2024-11-14T12:54:29,576 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open test,iii,1731588868017.03f289a2805f0b2f037581e9390ba191. 2024-11-14T12:54:29,576 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45309 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=669ef94e8b26,44819,1731588865173, table=test, region=22b9f10374fc1ea7a876d476ce0b083d. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-14T12:54:29,576 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => 03f289a2805f0b2f037581e9390ba191, NAME => 'test,iii,1731588868017.03f289a2805f0b2f037581e9390ba191.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-14T12:54:29,576 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 03f289a2805f0b2f037581e9390ba191 2024-11-14T12:54:29,576 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated test,iii,1731588868017.03f289a2805f0b2f037581e9390ba191.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:29,577 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for 03f289a2805f0b2f037581e9390ba191 2024-11-14T12:54:29,577 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for 03f289a2805f0b2f037581e9390ba191 2024-11-14T12:54:29,579 INFO [StoreOpener-03f289a2805f0b2f037581e9390ba191-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 03f289a2805f0b2f037581e9390ba191 2024-11-14T12:54:29,582 INFO [StoreOpener-03f289a2805f0b2f037581e9390ba191-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 03f289a2805f0b2f037581e9390ba191 columnFamilyName family 2024-11-14T12:54:29,582 DEBUG [StoreOpener-03f289a2805f0b2f037581e9390ba191-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T12:54:29,583 INFO [StoreOpener-03f289a2805f0b2f037581e9390ba191-1 {}] regionserver.HStore(327): Store=03f289a2805f0b2f037581e9390ba191/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T12:54:29,583 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for 03f289a2805f0b2f037581e9390ba191 2024-11-14T12:54:29,584 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=22 2024-11-14T12:54:29,584 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure a5d884ba0737bc572475e022a6ed1bc8, server=669ef94e8b26,43475,1731588864980 in 453 msec 2024-11-14T12:54:29,584 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/03f289a2805f0b2f037581e9390ba191 2024-11-14T12:54:29,586 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/03f289a2805f0b2f037581e9390ba191 2024-11-14T12:54:29,587 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for 03f289a2805f0b2f037581e9390ba191 2024-11-14T12:54:29,587 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for 03f289a2805f0b2f037581e9390ba191 2024-11-14T12:54:29,587 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a5d884ba0737bc572475e022a6ed1bc8, ASSIGN in 775 msec 2024-11-14T12:54:29,589 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=f8c0d62856832eec9f9ca20bb6db602c, regionState=OPEN, openSeqNum=2, regionLocation=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:29,592 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for 03f289a2805f0b2f037581e9390ba191 2024-11-14T12:54:29,595 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure f8c0d62856832eec9f9ca20bb6db602c, server=669ef94e8b26,43475,1731588864980 because future has completed 2024-11-14T12:54:29,597 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/03f289a2805f0b2f037581e9390ba191/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T12:54:29,598 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened 03f289a2805f0b2f037581e9390ba191; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62485859, jitterRate=-0.06888814270496368}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T12:54:29,598 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 03f289a2805f0b2f037581e9390ba191 2024-11-14T12:54:29,598 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for 03f289a2805f0b2f037581e9390ba191: Running coprocessor pre-open hook at 1731588869577Writing region info on filesystem at 1731588869577Initializing all the Stores at 1731588869578 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731588869578Cleaning up temporary data from old regions at 1731588869587 (+9 ms)Running coprocessor post-open hooks at 1731588869598 (+11 ms)Region opened successfully at 1731588869598 2024-11-14T12:54:29,599 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1731588868017.03f289a2805f0b2f037581e9390ba191., pid=51, masterSystemTime=1731588869230 2024-11-14T12:54:29,599 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=27 2024-11-14T12:54:29,600 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure 22b9f10374fc1ea7a876d476ce0b083d, server=669ef94e8b26,44819,1731588865173 in 580 msec 2024-11-14T12:54:29,602 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1731588868017.03f289a2805f0b2f037581e9390ba191. 2024-11-14T12:54:29,602 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=22b9f10374fc1ea7a876d476ce0b083d, ASSIGN in 791 msec 2024-11-14T12:54:29,602 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened test,iii,1731588868017.03f289a2805f0b2f037581e9390ba191. 2024-11-14T12:54:29,602 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(132): Open test,ddd,1731588868017.b8044551b940cf51bfa7c7633c4df21d. 2024-11-14T12:54:29,603 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7752): Opening region: {ENCODED => b8044551b940cf51bfa7c7633c4df21d, NAME => 'test,ddd,1731588868017.b8044551b940cf51bfa7c7633c4df21d.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-11-14T12:54:29,603 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test b8044551b940cf51bfa7c7633c4df21d 2024-11-14T12:54:29,603 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(898): Instantiated test,ddd,1731588868017.b8044551b940cf51bfa7c7633c4df21d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:29,603 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=03f289a2805f0b2f037581e9390ba191, regionState=OPEN, openSeqNum=2, regionLocation=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:29,603 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7794): checking encryption for b8044551b940cf51bfa7c7633c4df21d 2024-11-14T12:54:29,604 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7797): checking classloading for b8044551b940cf51bfa7c7633c4df21d 2024-11-14T12:54:29,607 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 03f289a2805f0b2f037581e9390ba191, server=669ef94e8b26,43475,1731588864980 because future has completed 2024-11-14T12:54:29,607 INFO [StoreOpener-b8044551b940cf51bfa7c7633c4df21d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region b8044551b940cf51bfa7c7633c4df21d 2024-11-14T12:54:29,610 INFO [StoreOpener-b8044551b940cf51bfa7c7633c4df21d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b8044551b940cf51bfa7c7633c4df21d columnFamilyName family 2024-11-14T12:54:29,610 DEBUG [StoreOpener-b8044551b940cf51bfa7c7633c4df21d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T12:54:29,611 INFO [StoreOpener-b8044551b940cf51bfa7c7633c4df21d-1 {}] regionserver.HStore(327): Store=b8044551b940cf51bfa7c7633c4df21d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T12:54:29,611 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1038): replaying wal for b8044551b940cf51bfa7c7633c4df21d 2024-11-14T12:54:29,613 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=25 2024-11-14T12:54:29,613 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/b8044551b940cf51bfa7c7633c4df21d 2024-11-14T12:54:29,613 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure f8c0d62856832eec9f9ca20bb6db602c, server=669ef94e8b26,43475,1731588864980 in 531 msec 2024-11-14T12:54:29,613 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/b8044551b940cf51bfa7c7633c4df21d 2024-11-14T12:54:29,614 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1048): stopping wal replay for b8044551b940cf51bfa7c7633c4df21d 2024-11-14T12:54:29,614 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1060): Cleaning up temporary data for b8044551b940cf51bfa7c7633c4df21d 2024-11-14T12:54:29,616 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f8c0d62856832eec9f9ca20bb6db602c, ASSIGN in 804 msec 2024-11-14T12:54:29,617 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): writing seq id for b8044551b940cf51bfa7c7633c4df21d 2024-11-14T12:54:29,621 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/b8044551b940cf51bfa7c7633c4df21d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T12:54:29,622 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=13 2024-11-14T12:54:29,622 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure 03f289a2805f0b2f037581e9390ba191, server=669ef94e8b26,43475,1731588864980 in 490 msec 2024-11-14T12:54:29,622 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1114): Opened b8044551b940cf51bfa7c7633c4df21d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66195472, jitterRate=-0.013610601425170898}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T12:54:29,623 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b8044551b940cf51bfa7c7633c4df21d 2024-11-14T12:54:29,623 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1006): Region open journal for b8044551b940cf51bfa7c7633c4df21d: Running coprocessor pre-open hook at 1731588869604Writing region info on filesystem at 1731588869604Initializing all the Stores at 1731588869606 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731588869606Cleaning up temporary data from old regions at 1731588869614 (+8 ms)Running coprocessor post-open hooks at 1731588869623 (+9 ms)Region opened successfully at 1731588869623 2024-11-14T12:54:29,624 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ddd,1731588868017.b8044551b940cf51bfa7c7633c4df21d., pid=52, masterSystemTime=1731588869230 2024-11-14T12:54:29,625 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=03f289a2805f0b2f037581e9390ba191, ASSIGN in 814 msec 2024-11-14T12:54:29,627 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=c8ddc4f331afb3a3007ef5c49cdf6ab8, regionState=OPEN, openSeqNum=2, regionLocation=669ef94e8b26,44819,1731588865173 2024-11-14T12:54:29,628 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ddd,1731588868017.b8044551b940cf51bfa7c7633c4df21d. 2024-11-14T12:54:29,628 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(153): Opened test,ddd,1731588868017.b8044551b940cf51bfa7c7633c4df21d. 2024-11-14T12:54:29,631 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure c8ddc4f331afb3a3007ef5c49cdf6ab8, server=669ef94e8b26,44819,1731588865173 because future has completed 2024-11-14T12:54:29,632 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=b8044551b940cf51bfa7c7633c4df21d, regionState=OPEN, openSeqNum=2, regionLocation=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:29,635 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure b8044551b940cf51bfa7c7633c4df21d, server=669ef94e8b26,43475,1731588864980 because future has completed 2024-11-14T12:54:29,642 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=8 2024-11-14T12:54:29,642 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure b8044551b940cf51bfa7c7633c4df21d, server=669ef94e8b26,43475,1731588864980 in 504 msec 2024-11-14T12:54:29,652 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=b8044551b940cf51bfa7c7633c4df21d, ASSIGN in 833 msec 2024-11-14T12:54:29,657 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=24 2024-11-14T12:54:29,657 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure c8ddc4f331afb3a3007ef5c49cdf6ab8, server=669ef94e8b26,44819,1731588865173 in 589 msec 2024-11-14T12:54:29,660 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN in 848 msec 2024-11-14T12:54:29,661 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=cadcf21958eaec53ee489ca0918e3ab7, regionState=OPEN, openSeqNum=2, regionLocation=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:29,664 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure cadcf21958eaec53ee489ca0918e3ab7, server=669ef94e8b26,46877,1731588865107 because future has completed 2024-11-14T12:54:29,667 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=333a3eba2bb2d318c041beb4642f5ef3, regionState=OPEN, openSeqNum=2, regionLocation=669ef94e8b26,44819,1731588865173 2024-11-14T12:54:29,671 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,44819,1731588865173 because future has completed 2024-11-14T12:54:29,673 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=28 2024-11-14T12:54:29,674 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=28, state=SUCCESS, hasLock=false; OpenRegionProcedure cadcf21958eaec53ee489ca0918e3ab7, server=669ef94e8b26,46877,1731588865107 in 684 msec 2024-11-14T12:54:29,676 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=cadcf21958eaec53ee489ca0918e3ab7, ASSIGN in 865 msec 2024-11-14T12:54:29,684 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=15 2024-11-14T12:54:29,684 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,44819,1731588865173 in 581 msec 2024-11-14T12:54:29,688 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=333a3eba2bb2d318c041beb4642f5ef3, ASSIGN in 875 msec 2024-11-14T12:54:29,690 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=cdc4b47ed9fe07b56101f9e0e17abcd7, regionState=OPEN, openSeqNum=2, regionLocation=669ef94e8b26,44819,1731588865173 2024-11-14T12:54:29,692 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure cdc4b47ed9fe07b56101f9e0e17abcd7, server=669ef94e8b26,44819,1731588865173 because future has completed 2024-11-14T12:54:29,693 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=bf47be17fd22bc204db28da07b30af2c, regionState=OPEN, openSeqNum=2, regionLocation=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:29,696 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure bf47be17fd22bc204db28da07b30af2c, server=669ef94e8b26,43475,1731588864980 because future has completed 2024-11-14T12:54:29,699 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=12 2024-11-14T12:54:29,699 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure cdc4b47ed9fe07b56101f9e0e17abcd7, server=669ef94e8b26,44819,1731588865173 in 625 msec 2024-11-14T12:54:29,702 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN in 891 msec 2024-11-14T12:54:29,707 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=10 2024-11-14T12:54:29,707 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure bf47be17fd22bc204db28da07b30af2c, server=669ef94e8b26,43475,1731588864980 in 565 msec 2024-11-14T12:54:29,710 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=4 2024-11-14T12:54:29,710 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=bf47be17fd22bc204db28da07b30af2c, ASSIGN in 899 msec 2024-11-14T12:54:29,712 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T12:54:29,712 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731588869712"}]},"ts":"1731588869712"} 2024-11-14T12:54:29,715 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLED in hbase:meta 2024-11-14T12:54:29,717 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T12:54:29,725 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test in 1.6920 sec 2024-11-14T12:54:30,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45309 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T12:54:30,205 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table test get assigned. Timeout = 60000ms 2024-11-14T12:54:30,205 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test completed 2024-11-14T12:54:30,206 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-14T12:54:30,228 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table test assigned to meta. Checking AM states. 2024-11-14T12:54:30,229 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-14T12:54:30,230 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table test assigned. 2024-11-14T12:54:30,231 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-14T12:54:30,237 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-14T12:54:30,264 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-14T12:54:30,285 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T12:54:30,288 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53382, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T12:54:30,290 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T12:54:30,293 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52336, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T12:54:30,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45309 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c2ccfc8c17ab1571bc4ea6d526017f9f changed from -1.0 to 0.0, refreshing cache 2024-11-14T12:54:30,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45309 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region b8044551b940cf51bfa7c7633c4df21d changed from -1.0 to 0.0, refreshing cache 2024-11-14T12:54:30,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45309 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region bf47be17fd22bc204db28da07b30af2c changed from -1.0 to 0.0, refreshing cache 2024-11-14T12:54:30,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45309 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 03f289a2805f0b2f037581e9390ba191 changed from -1.0 to 0.0, refreshing cache 2024-11-14T12:54:30,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45309 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 64b5bd4e6112417a55e871b36551c5b8 changed from -1.0 to 0.0, refreshing cache 2024-11-14T12:54:30,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45309 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 338b47e35e9a8b9dd3045d365aaf47bd changed from -1.0 to 0.0, refreshing cache 2024-11-14T12:54:30,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45309 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region a5d884ba0737bc572475e022a6ed1bc8 changed from -1.0 to 0.0, refreshing cache 2024-11-14T12:54:30,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45309 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f8c0d62856832eec9f9ca20bb6db602c changed from -1.0 to 0.0, refreshing cache 2024-11-14T12:54:30,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45309 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 66b34b9be40614bd68d0cea995e44193 changed from -1.0 to 0.0, refreshing cache 2024-11-14T12:54:30,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45309 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region a572ae08c8738aa198c40f32aa4b5f96 changed from -1.0 to 0.0, refreshing cache 2024-11-14T12:54:30,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45309 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 23f7fbc112f1353900c1181e3ed36707 changed from -1.0 to 0.0, refreshing cache 2024-11-14T12:54:30,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45309 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region cdc4b47ed9fe07b56101f9e0e17abcd7 changed from -1.0 to 0.0, refreshing cache 2024-11-14T12:54:30,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45309 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 333a3eba2bb2d318c041beb4642f5ef3 changed from -1.0 to 0.0, refreshing cache 2024-11-14T12:54:30,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45309 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 4f16209fa220cf1b7a18b891e4fe53c6 changed from -1.0 to 0.0, refreshing cache 2024-11-14T12:54:30,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45309 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 18ddd925c95b1b23296976da04f97e83 changed from -1.0 to 0.0, refreshing cache 2024-11-14T12:54:30,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45309 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c8ddc4f331afb3a3007ef5c49cdf6ab8 changed from -1.0 to 0.0, refreshing cache 2024-11-14T12:54:30,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45309 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 22b9f10374fc1ea7a876d476ce0b083d changed from -1.0 to 0.0, refreshing cache 2024-11-14T12:54:30,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45309 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 2201f29440b3aa3259741bbb374e209d changed from -1.0 to 0.0, refreshing cache 2024-11-14T12:54:30,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45309 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 0fd0512a8a2f36befd3453e1140df3af changed from -1.0 to 0.0, refreshing cache 2024-11-14T12:54:30,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45309 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f76ec25f51f69b1106342a7dfae65fb6 changed from -1.0 to 0.0, refreshing cache 2024-11-14T12:54:30,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45309 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1358fa84af33d1799bd11ba7f4959a37 changed from -1.0 to 0.0, refreshing cache 2024-11-14T12:54:30,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45309 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region e90cc230c4f13fece446e3e8420dddbe changed from -1.0 to 0.0, refreshing cache 2024-11-14T12:54:30,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45309 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ea6e95809b33f853859fa227179bfe85 changed from -1.0 to 0.0, refreshing cache 2024-11-14T12:54:30,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45309 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1d33b9e9c25fcfc654d257646677381c changed from -1.0 to 0.0, refreshing cache 2024-11-14T12:54:30,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45309 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 87eb074dbd46b3bc6e27fb0bf3291d5b changed from -1.0 to 0.0, refreshing cache 2024-11-14T12:54:30,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45309 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region cadcf21958eaec53ee489ca0918e3ab7 changed from -1.0 to 0.0, refreshing cache 2024-11-14T12:54:30,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45309 {}] rsgroup.RSGroupBasedLoadBalancer(138): Balancing RSGroup=default 2024-11-14T12:54:30,351 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45309 {}] rsgroup.RSGroupBasedLoadBalancer(151): Start Generate Balance plan for group: default 2024-11-14T12:54:30,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45309 {}] balancer.BaseLoadBalancer(619): Start Generate Balance plan for cluster. 2024-11-14T12:54:30,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45309 {}] balancer.BalancerClusterState(204): Hosts are {669ef94e8b26=0} racks are {/default-rack=0} 2024-11-14T12:54:30,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45309 {}] balancer.BalancerClusterState(303): server 0 has 9 regions 2024-11-14T12:54:30,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45309 {}] balancer.BalancerClusterState(303): server 1 has 10 regions 2024-11-14T12:54:30,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45309 {}] balancer.BalancerClusterState(303): server 2 has 8 regions 2024-11-14T12:54:30,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45309 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-14T12:54:30,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45309 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-14T12:54:30,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45309 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-14T12:54:30,353 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45309 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-14T12:54:30,353 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45309 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-14T12:54:30,353 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45309 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-14T12:54:30,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45309 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1 2024-11-14T12:54:30,357 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45309 {}] balancer.StochasticLoadBalancer(403): Cluster wide - Calculating plan. may take up to 30000ms to complete. 2024-11-14T12:54:30,358 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45309 {}] balancer.StochasticLoadBalancer(515): Start StochasticLoadBalancer.balancer, initial weighted average imbalance=0.06931633435746164, functionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.06415002990995843, need balance); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.0); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); computedMaxSteps=64800 2024-11-14T12:54:30,949 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45309 {}] balancer.StochasticLoadBalancer(562): Finished computing new moving plan. Computation took 595 ms to try 64800 different iterations. Found a solution that moves 1 regions; Going from a computed imbalance of 0.06931633435746164 to a new imbalance of 0.01690158032826896. funtionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.0); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.037037037037037035, need balance); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); 2024-11-14T12:54:30,953 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45309 {}] master.HMaster(2167): Balancer plans size is 1, the balance interval is 300000 ms, and the max number regions in transition is 27 2024-11-14T12:54:30,953 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45309 {}] master.HMaster(2172): balance hri=23f7fbc112f1353900c1181e3ed36707, source=669ef94e8b26,44819,1731588865173, destination=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:30,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45309 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test, region=23f7fbc112f1353900c1181e3ed36707, REOPEN/MOVE 2024-11-14T12:54:30,956 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test, region=23f7fbc112f1353900c1181e3ed36707, REOPEN/MOVE 2024-11-14T12:54:30,958 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=23f7fbc112f1353900c1181e3ed36707, regionState=CLOSING, regionLocation=669ef94e8b26,44819,1731588865173 2024-11-14T12:54:30,961 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=test, region=23f7fbc112f1353900c1181e3ed36707, REOPEN/MOVE because future has completed 2024-11-14T12:54:30,961 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T12:54:30,961 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE, hasLock=false; CloseRegionProcedure 23f7fbc112f1353900c1181e3ed36707, server=669ef94e8b26,44819,1731588865173}] 2024-11-14T12:54:31,119 INFO [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(122): Close 23f7fbc112f1353900c1181e3ed36707 2024-11-14T12:54:31,119 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T12:54:31,121 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1722): Closing 23f7fbc112f1353900c1181e3ed36707, disabling compactions & flushes 2024-11-14T12:54:31,121 INFO [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1755): Closing region test,eee,1731588868017.23f7fbc112f1353900c1181e3ed36707. 2024-11-14T12:54:31,121 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1731588868017.23f7fbc112f1353900c1181e3ed36707. 2024-11-14T12:54:31,121 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1843): Acquired close lock on test,eee,1731588868017.23f7fbc112f1353900c1181e3ed36707. after waiting 0 ms 2024-11-14T12:54:31,121 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1853): Updates disabled for region test,eee,1731588868017.23f7fbc112f1353900c1181e3ed36707. 2024-11-14T12:54:31,126 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/23f7fbc112f1353900c1181e3ed36707/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T12:54:31,129 INFO [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1973): Closed test,eee,1731588868017.23f7fbc112f1353900c1181e3ed36707. 2024-11-14T12:54:31,129 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1676): Region close journal for 23f7fbc112f1353900c1181e3ed36707: Waiting for close lock at 1731588871121Running coprocessor pre-close hooks at 1731588871121Disabling compacts and flushes for region at 1731588871121Disabling writes for close at 1731588871121Writing region close event to WAL at 1731588871122 (+1 ms)Running coprocessor post-close hooks at 1731588871127 (+5 ms)Closed at 1731588871129 (+2 ms) 2024-11-14T12:54:31,130 INFO [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegionServer(3302): Adding 23f7fbc112f1353900c1181e3ed36707 move to 669ef94e8b26,46877,1731588865107 record at close sequenceid=2 2024-11-14T12:54:31,136 INFO [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(157): Closed 23f7fbc112f1353900c1181e3ed36707 2024-11-14T12:54:31,137 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=23f7fbc112f1353900c1181e3ed36707, regionState=CLOSED 2024-11-14T12:54:31,139 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE, hasLock=false; CloseRegionProcedure 23f7fbc112f1353900c1181e3ed36707, server=669ef94e8b26,44819,1731588865173 because future has completed 2024-11-14T12:54:31,143 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-14T12:54:31,143 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; CloseRegionProcedure 23f7fbc112f1353900c1181e3ed36707, server=669ef94e8b26,44819,1731588865173 in 180 msec 2024-11-14T12:54:31,144 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=23f7fbc112f1353900c1181e3ed36707, REOPEN/MOVE; state=CLOSED, location=669ef94e8b26,46877,1731588865107; forceNewPlan=false, retain=false 2024-11-14T12:54:31,295 INFO [669ef94e8b26:45309 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-14T12:54:31,295 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=23f7fbc112f1353900c1181e3ed36707, regionState=OPENING, regionLocation=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:31,298 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=23f7fbc112f1353900c1181e3ed36707, REOPEN/MOVE because future has completed 2024-11-14T12:54:31,298 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=57, state=RUNNABLE, hasLock=false; OpenRegionProcedure 23f7fbc112f1353900c1181e3ed36707, server=669ef94e8b26,46877,1731588865107}] 2024-11-14T12:54:31,456 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(132): Open test,eee,1731588868017.23f7fbc112f1353900c1181e3ed36707. 2024-11-14T12:54:31,457 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7752): Opening region: {ENCODED => 23f7fbc112f1353900c1181e3ed36707, NAME => 'test,eee,1731588868017.23f7fbc112f1353900c1181e3ed36707.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-14T12:54:31,457 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 23f7fbc112f1353900c1181e3ed36707 2024-11-14T12:54:31,457 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(898): Instantiated test,eee,1731588868017.23f7fbc112f1353900c1181e3ed36707.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:31,458 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7794): checking encryption for 23f7fbc112f1353900c1181e3ed36707 2024-11-14T12:54:31,458 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7797): checking classloading for 23f7fbc112f1353900c1181e3ed36707 2024-11-14T12:54:31,460 INFO [StoreOpener-23f7fbc112f1353900c1181e3ed36707-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 23f7fbc112f1353900c1181e3ed36707 2024-11-14T12:54:31,461 INFO [StoreOpener-23f7fbc112f1353900c1181e3ed36707-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 23f7fbc112f1353900c1181e3ed36707 columnFamilyName family 2024-11-14T12:54:31,461 DEBUG [StoreOpener-23f7fbc112f1353900c1181e3ed36707-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T12:54:31,461 INFO [StoreOpener-23f7fbc112f1353900c1181e3ed36707-1 {}] regionserver.HStore(327): Store=23f7fbc112f1353900c1181e3ed36707/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T12:54:31,462 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1038): replaying wal for 23f7fbc112f1353900c1181e3ed36707 2024-11-14T12:54:31,463 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/23f7fbc112f1353900c1181e3ed36707 2024-11-14T12:54:31,465 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/23f7fbc112f1353900c1181e3ed36707 2024-11-14T12:54:31,465 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1048): stopping wal replay for 23f7fbc112f1353900c1181e3ed36707 2024-11-14T12:54:31,465 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1060): Cleaning up temporary data for 23f7fbc112f1353900c1181e3ed36707 2024-11-14T12:54:31,468 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1093): writing seq id for 23f7fbc112f1353900c1181e3ed36707 2024-11-14T12:54:31,469 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1114): Opened 23f7fbc112f1353900c1181e3ed36707; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69283406, jitterRate=0.03240320086479187}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T12:54:31,469 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 23f7fbc112f1353900c1181e3ed36707 2024-11-14T12:54:31,470 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1006): Region open journal for 23f7fbc112f1353900c1181e3ed36707: Running coprocessor pre-open hook at 1731588871458Writing region info on filesystem at 1731588871458Initializing all the Stores at 1731588871459 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731588871459Cleaning up temporary data from old regions at 1731588871465 (+6 ms)Running coprocessor post-open hooks at 1731588871469 (+4 ms)Region opened successfully at 1731588871469 2024-11-14T12:54:31,471 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1731588868017.23f7fbc112f1353900c1181e3ed36707., pid=59, masterSystemTime=1731588871450 2024-11-14T12:54:31,474 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1731588868017.23f7fbc112f1353900c1181e3ed36707. 2024-11-14T12:54:31,474 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(153): Opened test,eee,1731588868017.23f7fbc112f1353900c1181e3ed36707. 2024-11-14T12:54:31,474 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=23f7fbc112f1353900c1181e3ed36707, regionState=OPEN, openSeqNum=5, regionLocation=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:31,477 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=57, state=RUNNABLE, hasLock=false; OpenRegionProcedure 23f7fbc112f1353900c1181e3ed36707, server=669ef94e8b26,46877,1731588865107 because future has completed 2024-11-14T12:54:31,481 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=57 2024-11-14T12:54:31,481 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=57, state=SUCCESS, hasLock=false; OpenRegionProcedure 23f7fbc112f1353900c1181e3ed36707, server=669ef94e8b26,46877,1731588865107 in 180 msec 2024-11-14T12:54:31,483 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=23f7fbc112f1353900c1181e3ed36707, REOPEN/MOVE in 528 msec 2024-11-14T12:54:31,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45309 {}] master.HMaster(2203): Balancer is going into sleep until next period in 300000ms 2024-11-14T12:54:31,558 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [900,000] milli-secs(wait.for.ratio=[1]) 2024-11-14T12:54:31,570 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45309 {}] master.MasterRpcServices(567): Client=jenkins//172.17.0.3 set balanceSwitch=false 2024-11-14T12:54:31,587 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.assignment.TestRollbackSCP#testFailAndRollback Thread=387, OpenFileDescriptor=645, MaxFileDescriptor=1048576, SystemLoadAverage=263, ProcessCount=11, AvailableMemoryMB=7493 2024-11-14T12:54:31,589 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table hbase:meta 2024-11-14T12:54:31,590 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=hbase:meta,,1 2024-11-14T12:54:31,590 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing 669ef94e8b26,44819,1731588865173 2024-11-14T12:54:31,591 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 669ef94e8b26,44819,1731588865173: Simulated kill ***** 2024-11-14T12:54:31,591 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-14T12:54:31,593 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-14T12:54:31,595 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-14T12:54:31,596 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-14T12:54:31,596 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-14T12:54:31,596 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 346800320 }, "NonHeapMemoryUsage": { "committed": 154271744, "init": 7667712, "max": -1, "used": 151907640 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "669ef94e8b26" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "669ef94e8b26" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.Context": "regionserver", "tag.Hostname": "669ef94e8b26" } ] } 2024-11-14T12:54:31,599 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45309 {}] master.MasterRpcServices(700): 669ef94e8b26,44819,1731588865173 reported a fatal error: ***** ABORTING region server 669ef94e8b26,44819,1731588865173: Simulated kill ***** 2024-11-14T12:54:31,600 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '669ef94e8b26,44819,1731588865173' ***** 2024-11-14T12:54:31,600 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-11-14T12:54:31,601 INFO [RS:2;669ef94e8b26:44819 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-14T12:54:31,601 INFO [RS:2;669ef94e8b26:44819 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-14T12:54:31,601 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-14T12:54:31,601 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [15,000] milli-secs(wait.for.ratio=[1]) 2024-11-14T12:54:31,601 INFO [RS:2;669ef94e8b26:44819 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-14T12:54:31,601 INFO [RS:2;669ef94e8b26:44819 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T12:54:31,601 INFO [RS:2;669ef94e8b26:44819 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;669ef94e8b26:44819. 2024-11-14T12:54:31,602 DEBUG [RS:2;669ef94e8b26:44819 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T12:54:31,606 DEBUG [RS:2;669ef94e8b26:44819 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T12:54:31,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43871 is added to blk_1073741836_1012 (size=33749) 2024-11-14T12:54:31,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36021 is added to blk_1073741836_1012 (size=33749) 2024-11-14T12:54:31,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741836_1012 (size=33749) 2024-11-14T12:54:31,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36021 is added to blk_1073741833_1009 (size=2769) 2024-11-14T12:54:31,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741833_1009 (size=2769) 2024-11-14T12:54:31,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43871 is added to blk_1073741833_1009 (size=2769) 2024-11-14T12:54:31,619 DEBUG [RS:2;669ef94e8b26:44819 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T12:54:31,619 INFO [RS:2;669ef94e8b26:44819 {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T12:54:31,619 INFO [RS:2;669ef94e8b26:44819 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:44819 2024-11-14T12:54:31,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44819-0x1013ab2e72b0003, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/669ef94e8b26,44819,1731588865173 2024-11-14T12:54:31,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45309-0x1013ab2e72b0000, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T12:54:31,664 INFO [RS:2;669ef94e8b26:44819 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T12:54:31,672 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [669ef94e8b26,44819,1731588865173] 2024-11-14T12:54:31,678 INFO [regionserver/669ef94e8b26:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T12:54:31,680 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/669ef94e8b26,44819,1731588865173 already deleted, retry=false 2024-11-14T12:54:31,680 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 669ef94e8b26,44819,1731588865173 on 669ef94e8b26,45309,1731588864185 2024-11-14T12:54:31,685 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true 2024-11-14T12:54:31,689 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=60, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true 2024-11-14T12:54:31,689 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1991): Scheduled ServerCrashProcedure pid=60 for 669ef94e8b26,44819,1731588865173 (carryingMeta=true) 669ef94e8b26,44819,1731588865173/CRASHED/regionCount=9/lock=java.util.concurrent.locks.ReentrantReadWriteLock@6b332840[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-14T12:54:31,690 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=60, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true, isMeta: true 2024-11-14T12:54:31,692 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/WALs/669ef94e8b26,44819,1731588865173-splitting 2024-11-14T12:54:31,694 INFO [PEWorker-1 {}] master.SplitWALManager(105): 669ef94e8b26,44819,1731588865173 WAL count=1, meta=true 2024-11-14T12:54:31,696 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta}] 2024-11-14T12:54:31,702 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:31,702 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-14T12:54:31,704 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107}] 2024-11-14T12:54:31,772 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44819-0x1013ab2e72b0003, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T12:54:31,772 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44819-0x1013ab2e72b0003, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T12:54:31,773 INFO [RS:2;669ef94e8b26:44819 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T12:54:31,773 INFO [RS:2;669ef94e8b26:44819 {}] regionserver.HRegionServer(1031): Exiting; stopping=669ef94e8b26,44819,1731588865173; zookeeper connection closed. 2024-11-14T12:54:31,773 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@758b6549 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@758b6549 2024-11-14T12:54:31,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=62 2024-11-14T12:54:31,869 INFO [regionserver/669ef94e8b26:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-14T12:54:31,870 INFO [regionserver/669ef94e8b26:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-14T12:54:31,888 INFO [RS_LOG_REPLAY_OPS-regionserver/669ef94e8b26:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(299): Splitting hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/WALs/669ef94e8b26,44819,1731588865173-splitting/669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, size=33.0 K (33749bytes) 2024-11-14T12:54:31,889 DEBUG [RS_LOG_REPLAY_OPS-regionserver/669ef94e8b26:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-14T12:54:31,889 DEBUG [RS_LOG_REPLAY_OPS-regionserver/669ef94e8b26:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-14T12:54:31,889 INFO [RS_LOG_REPLAY_OPS-regionserver/669ef94e8b26:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/WALs/669ef94e8b26,44819,1731588865173-splitting/669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta 2024-11-14T12:54:31,894 INFO [RS_LOG_REPLAY_OPS-regionserver/669ef94e8b26:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/WALs/669ef94e8b26,44819,1731588865173-splitting/669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta after 3ms 2024-11-14T12:54:31,901 INFO [RS_LOG_REPLAY_OPS-regionserver/669ef94e8b26:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(310): Open hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/WALs/669ef94e8b26,44819,1731588865173-splitting/669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta took 15ms 2024-11-14T12:54:31,909 DEBUG [RS_LOG_REPLAY_OPS-regionserver/669ef94e8b26:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-14T12:54:31,915 INFO [RS_LOG_REPLAY_OPS-regionserver/669ef94e8b26:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta.temp 2024-11-14T12:54:31,919 INFO [RS_LOG_REPLAY_OPS-regionserver/669ef94e8b26:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta.temp 2024-11-14T12:54:31,919 DEBUG [RS_LOG_REPLAY_OPS-regionserver/669ef94e8b26:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/WALs/669ef94e8b26,44819,1731588865173-splitting/669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta so closing down 2024-11-14T12:54:31,920 DEBUG [RS_LOG_REPLAY_OPS-regionserver/669ef94e8b26:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T12:54:31,924 INFO [RS_LOG_REPLAY_OPS-regionserver/669ef94e8b26:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T12:54:31,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36021 is added to blk_1073741864_1040 (size=33508) 2024-11-14T12:54:31,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43871 is added to blk_1073741864_1040 (size=33508) 2024-11-14T12:54:31,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741864_1040 (size=33508) 2024-11-14T12:54:31,939 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta.temp (wrote 61 edits, skipped 0 edits in 3 ms) 2024-11-14T12:54:31,941 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta.temp to hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-14T12:54:31,944 INFO [RS_LOG_REPLAY_OPS-regionserver/669ef94e8b26:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(425): Processed 62 edits across 1 Regions in 39 ms; skipped=1; WAL=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/WALs/669ef94e8b26,44819,1731588865173-splitting/669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, size=33.0 K, length=33749, corrupted=false, cancelled=false 2024-11-14T12:54:31,944 DEBUG [RS_LOG_REPLAY_OPS-regionserver/669ef94e8b26:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(428): Completed split of hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/WALs/669ef94e8b26,44819,1731588865173-splitting/669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, journal: Splitting hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/WALs/669ef94e8b26,44819,1731588865173-splitting/669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, size=33.0 K (33749bytes) at 1731588871888Creating recovered edits writer path=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta.temp at 1731588871919 (+31 ms)Finishing writing output for hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/WALs/669ef94e8b26,44819,1731588865173-splitting/669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta so closing down at 17315888719193 split writer threads finished at 1731588871924 (+5 ms)Closed recovered edits writer path=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta.temp (wrote 61 edits, skipped 0 edits in 3 ms) at 1731588871939 (+15 ms)Rename recovered edits hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta.temp to hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 at 1731588871941 (+2 ms)Processed 62 edits across 1 Regions in 39 ms; skipped=1; WAL=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/WALs/669ef94e8b26,44819,1731588865173-splitting/669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, size=33.0 K, length=33749, corrupted=false, cancelled=false at 1731588871944 (+3 ms) 2024-11-14T12:54:31,945 DEBUG [RS_LOG_REPLAY_OPS-regionserver/669ef94e8b26:0-0 {event_type=RS_LOG_REPLAY, pid=62}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/WALs/669ef94e8b26,44819,1731588865173-splitting/669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta 2024-11-14T12:54:31,946 DEBUG [RS_LOG_REPLAY_OPS-regionserver/669ef94e8b26:0-0 {event_type=RS_LOG_REPLAY, pid=62}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=62 2024-11-14T12:54:31,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45309 {}] master.HMaster(4169): Remote procedure done, pid=62 2024-11-14T12:54:31,956 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/WALs/669ef94e8b26,44819,1731588865173-splitting/669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta to hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/oldWALs 2024-11-14T12:54:31,959 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-14T12:54:31,959 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 in 252 msec 2024-11-14T12:54:31,961 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:31,964 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-14T12:54:31,964 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 in 266 msec 2024-11-14T12:54:31,966 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/WALs/669ef94e8b26,44819,1731588865173-splitting dir is empty, no logs to split. 2024-11-14T12:54:31,967 INFO [PEWorker-1 {}] master.SplitWALManager(105): 669ef94e8b26,44819,1731588865173 WAL count=0, meta=true 2024-11-14T12:54:31,967 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 669ef94e8b26,44819,1731588865173 WAL splitting is done? wals=0, meta=true 2024-11-14T12:54:31,969 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-14T12:54:31,971 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T12:54:31,972 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-14T12:54:32,122 DEBUG [669ef94e8b26:45309 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-14T12:54:32,123 DEBUG [669ef94e8b26:45309 {}] balancer.BalancerClusterState(204): Hosts are {669ef94e8b26=0} racks are {/default-rack=0} 2024-11-14T12:54:32,123 DEBUG [669ef94e8b26:45309 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-14T12:54:32,123 DEBUG [669ef94e8b26:45309 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-14T12:54:32,123 DEBUG [669ef94e8b26:45309 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-14T12:54:32,123 DEBUG [669ef94e8b26:45309 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-14T12:54:32,124 INFO [669ef94e8b26:45309 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-14T12:54:32,124 INFO [669ef94e8b26:45309 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-14T12:54:32,124 DEBUG [669ef94e8b26:45309 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-14T12:54:32,124 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=63 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:32,126 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 669ef94e8b26,43475,1731588864980, state=OPENING 2024-11-14T12:54:32,156 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46877-0x1013ab2e72b0002, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:32,156 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45309-0x1013ab2e72b0000, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:32,156 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43475-0x1013ab2e72b0001, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:32,156 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:32,156 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:32,156 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:32,156 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T12:54:32,157 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=669ef94e8b26,43475,1731588864980}] 2024-11-14T12:54:32,319 INFO [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=64}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-14T12:54:32,319 INFO [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T12:54:32,320 INFO [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=64}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-14T12:54:32,323 INFO [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=669ef94e8b26%2C43475%2C1731588864980.meta, suffix=.meta, logDir=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/WALs/669ef94e8b26,43475,1731588864980, archiveDir=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/oldWALs, maxLogs=32 2024-11-14T12:54:32,339 DEBUG [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=64}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/WALs/669ef94e8b26,43475,1731588864980/669ef94e8b26%2C43475%2C1731588864980.meta.1731588872325.meta, exclude list is [], retry=0 2024-11-14T12:54:32,342 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36021,DS-90f4821d-4e9a-4239-9e98-c91b432844d1,DISK] 2024-11-14T12:54:32,342 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43871,DS-9aae07ee-1142-4a11-b0be-2c50c0112df3,DISK] 2024-11-14T12:54:32,342 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35109,DS-111c8733-6272-4257-b572-55835d3a523b,DISK] 2024-11-14T12:54:32,347 INFO [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/WALs/669ef94e8b26,43475,1731588864980/669ef94e8b26%2C43475%2C1731588864980.meta.1731588872325.meta 2024-11-14T12:54:32,348 DEBUG [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44851:44851),(127.0.0.1/127.0.0.1:38479:38479),(127.0.0.1/127.0.0.1:39561:39561)] 2024-11-14T12:54:32,348 DEBUG [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-14T12:54:32,348 DEBUG [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=64}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-14T12:54:32,348 DEBUG [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-14T12:54:32,348 INFO [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-14T12:54:32,349 DEBUG [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-14T12:54:32,349 DEBUG [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:32,349 DEBUG [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-14T12:54:32,349 DEBUG [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-14T12:54:32,351 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-14T12:54:32,352 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T12:54:32,352 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T12:54:32,352 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T12:54:32,352 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-14T12:54:32,353 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T12:54:32,353 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T12:54:32,354 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T12:54:32,354 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-14T12:54:32,355 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T12:54:32,355 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T12:54:32,356 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T12:54:32,356 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-14T12:54:32,357 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T12:54:32,357 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T12:54:32,357 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T12:54:32,358 DEBUG [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T12:54:32,359 DEBUG [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/hbase/meta/1588230740 2024-11-14T12:54:32,361 DEBUG [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/hbase/meta/1588230740 2024-11-14T12:54:32,363 INFO [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-14T12:54:32,377 DEBUG [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5793): Applied 277, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=64, path=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-14T12:54:32,380 INFO [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=29.41 KB heapSize=50.67 KB 2024-11-14T12:54:32,445 DEBUG [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/hbase/meta/1588230740/.tmp/info/2986975c98094cebb05785bea48305ab is 138, key is test,bbb,1731588868017.c2ccfc8c17ab1571bc4ea6d526017f9f./info:regioninfo/1731588869496/Put/seqid=0 2024-11-14T12:54:32,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43871 is added to blk_1073741866_1042 (size=36385) 2024-11-14T12:54:32,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741866_1042 (size=36385) 2024-11-14T12:54:32,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36021 is added to blk_1073741866_1042 (size=36385) 2024-11-14T12:54:32,460 INFO [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.26 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/hbase/meta/1588230740/.tmp/info/2986975c98094cebb05785bea48305ab 2024-11-14T12:54:32,533 DEBUG [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/hbase/meta/1588230740/.tmp/ns/6213e6b61bfa4ab98e7567c3f7280b3d is 43, key is default/ns:d/1731588867681/Put/seqid=0 2024-11-14T12:54:32,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43871 is added to blk_1073741867_1043 (size=5153) 2024-11-14T12:54:32,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741867_1043 (size=5153) 2024-11-14T12:54:32,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36021 is added to blk_1073741867_1043 (size=5153) 2024-11-14T12:54:32,948 INFO [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/hbase/meta/1588230740/.tmp/ns/6213e6b61bfa4ab98e7567c3f7280b3d 2024-11-14T12:54:32,975 DEBUG [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/hbase/meta/1588230740/.tmp/table/7e553afa5414422ab6217bd0534e6113 is 40, key is test/table:state/1731588869712/Put/seqid=0 2024-11-14T12:54:32,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36021 is added to blk_1073741868_1044 (size=5165) 2024-11-14T12:54:32,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741868_1044 (size=5165) 2024-11-14T12:54:32,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43871 is added to blk_1073741868_1044 (size=5165) 2024-11-14T12:54:32,984 INFO [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=72 B at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/hbase/meta/1588230740/.tmp/table/7e553afa5414422ab6217bd0534e6113 2024-11-14T12:54:32,994 DEBUG [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/hbase/meta/1588230740/.tmp/info/2986975c98094cebb05785bea48305ab as hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/hbase/meta/1588230740/info/2986975c98094cebb05785bea48305ab 2024-11-14T12:54:33,014 INFO [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/hbase/meta/1588230740/info/2986975c98094cebb05785bea48305ab, entries=273, sequenceid=64, filesize=35.5 K 2024-11-14T12:54:33,017 DEBUG [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/hbase/meta/1588230740/.tmp/ns/6213e6b61bfa4ab98e7567c3f7280b3d as hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/hbase/meta/1588230740/ns/6213e6b61bfa4ab98e7567c3f7280b3d 2024-11-14T12:54:33,026 INFO [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/hbase/meta/1588230740/ns/6213e6b61bfa4ab98e7567c3f7280b3d, entries=2, sequenceid=64, filesize=5.0 K 2024-11-14T12:54:33,028 DEBUG [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/hbase/meta/1588230740/.tmp/table/7e553afa5414422ab6217bd0534e6113 as hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/hbase/meta/1588230740/table/7e553afa5414422ab6217bd0534e6113 2024-11-14T12:54:33,039 INFO [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/hbase/meta/1588230740/table/7e553afa5414422ab6217bd0534e6113, entries=2, sequenceid=64, filesize=5.0 K 2024-11-14T12:54:33,044 INFO [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(3140): Finished flush of dataSize ~29.41 KB/30111, heapSize ~50.38 KB/51584, currentSize=0 B/0 for 1588230740 in 662ms, sequenceid=64, compaction requested=false; wal=null 2024-11-14T12:54:33,046 DEBUG [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-14T12:54:33,049 DEBUG [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-14T12:54:33,050 DEBUG [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T12:54:33,050 DEBUG [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T12:54:33,051 DEBUG [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-14T12:54:33,053 DEBUG [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T12:54:33,054 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T12:54:33,056 DEBUG [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/hbase/meta/1588230740/recovered.edits/64.seqid, newMaxSeqId=64, maxSeqId=1 2024-11-14T12:54:33,058 INFO [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=65; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61946669, jitterRate=-0.0769226998090744}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-14T12:54:33,058 DEBUG [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-14T12:54:33,058 DEBUG [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731588872349Writing region info on filesystem at 1731588872349Initializing all the Stores at 1731588872350 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731588872350Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731588872350Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731588872350Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731588872351 (+1 ms)Obtaining lock to block concurrent updates at 1731588872380 (+29 ms)Preparing flush snapshotting stores in 1588230740 at 1731588872381 (+1 ms)Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=30111, getHeapSize=51824, getOffHeapSize=0, getCellsCount=277 at 1731588872383 (+2 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731588872383Flushing 1588230740/info: creating writer at 1731588872385 (+2 ms)Flushing 1588230740/info: appending metadata at 1731588872435 (+50 ms)Flushing 1588230740/info: closing flushed file at 1731588872438 (+3 ms)Flushing 1588230740/ns: creating writer at 1731588872517 (+79 ms)Flushing 1588230740/ns: appending metadata at 1731588872532 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1731588872532Flushing 1588230740/table: creating writer at 1731588872958 (+426 ms)Flushing 1588230740/table: appending metadata at 1731588872974 (+16 ms)Flushing 1588230740/table: closing flushed file at 1731588872974Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4fb0a0b7: reopening flushed file at 1731588872992 (+18 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2bd8f8cd: reopening flushed file at 1731588873016 (+24 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60023360: reopening flushed file at 1731588873027 (+11 ms)Finished flush of dataSize ~29.41 KB/30111, heapSize ~50.38 KB/51584, currentSize=0 B/0 for 1588230740 in 662ms, sequenceid=64, compaction requested=false; wal=null at 1731588873044 (+17 ms)Cleaning up temporary data from old regions at 1731588873051 (+7 ms)Running coprocessor post-open hooks at 1731588873058 (+7 ms)Region opened successfully at 1731588873058 2024-11-14T12:54:33,060 INFO [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=64, masterSystemTime=1731588872311 2024-11-14T12:54:33,063 DEBUG [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-14T12:54:33,063 INFO [RS_OPEN_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_META, pid=64}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-14T12:54:33,065 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=63 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=65, regionLocation=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:33,072 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 669ef94e8b26,43475,1731588864980, state=OPEN 2024-11-14T12:54:33,088 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45309 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=669ef94e8b26,43475,1731588864980, table=hbase:meta, region=1588230740. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-14T12:54:33,097 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43475-0x1013ab2e72b0001, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:33,097 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46877-0x1013ab2e72b0002, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:33,097 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45309-0x1013ab2e72b0000, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:33,097 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:33,097 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:33,097 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=64, ppid=63, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:33,097 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:33,103 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-14T12:54:33,103 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=669ef94e8b26,43475,1731588864980 in 941 msec 2024-11-14T12:54:33,108 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=60 2024-11-14T12:54:33,108 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1.1340 sec 2024-11-14T12:54:33,109 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 669ef94e8b26,44819,1731588865173 had 9 regions 2024-11-14T12:54:33,110 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=60, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true, isMeta: false 2024-11-14T12:54:33,113 INFO [PEWorker-1 {}] master.SplitWALManager(105): 669ef94e8b26,44819,1731588865173 WAL count=1, meta=false 2024-11-14T12:54:33,114 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=60, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078}] 2024-11-14T12:54:33,117 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:33,118 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107}] 2024-11-14T12:54:33,119 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test' 2024-11-14T12:54:33,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46877 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=66 2024-11-14T12:54:33,292 INFO [RS_LOG_REPLAY_OPS-regionserver/669ef94e8b26:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(299): Splitting hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/WALs/669ef94e8b26,44819,1731588865173-splitting/669ef94e8b26%2C44819%2C1731588865173.1731588867078, size=2.7 K (2769bytes) 2024-11-14T12:54:33,293 INFO [RS_LOG_REPLAY_OPS-regionserver/669ef94e8b26:0-1 {event_type=RS_LOG_REPLAY, pid=66}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/WALs/669ef94e8b26,44819,1731588865173-splitting/669ef94e8b26%2C44819%2C1731588865173.1731588867078 2024-11-14T12:54:33,293 INFO [RS_LOG_REPLAY_OPS-regionserver/669ef94e8b26:0-1 {event_type=RS_LOG_REPLAY, pid=66}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/WALs/669ef94e8b26,44819,1731588865173-splitting/669ef94e8b26%2C44819%2C1731588865173.1731588867078 after 0ms 2024-11-14T12:54:33,297 INFO [RS_LOG_REPLAY_OPS-regionserver/669ef94e8b26:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(310): Open hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/WALs/669ef94e8b26,44819,1731588865173-splitting/669ef94e8b26%2C44819%2C1731588865173.1731588867078 took 5ms 2024-11-14T12:54:33,305 DEBUG [RS_LOG_REPLAY_OPS-regionserver/669ef94e8b26:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 4f16209fa220cf1b7a18b891e4fe53c6: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-14T12:54:33,307 DEBUG [RS_LOG_REPLAY_OPS-regionserver/669ef94e8b26:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 18ddd925c95b1b23296976da04f97e83: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-14T12:54:33,309 DEBUG [RS_LOG_REPLAY_OPS-regionserver/669ef94e8b26:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 2201f29440b3aa3259741bbb374e209d: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-14T12:54:33,311 DEBUG [RS_LOG_REPLAY_OPS-regionserver/669ef94e8b26:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for cdc4b47ed9fe07b56101f9e0e17abcd7: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-14T12:54:33,313 DEBUG [RS_LOG_REPLAY_OPS-regionserver/669ef94e8b26:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 333a3eba2bb2d318c041beb4642f5ef3: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-14T12:54:33,315 DEBUG [RS_LOG_REPLAY_OPS-regionserver/669ef94e8b26:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 23f7fbc112f1353900c1181e3ed36707: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-14T12:54:33,317 DEBUG [RS_LOG_REPLAY_OPS-regionserver/669ef94e8b26:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 22b9f10374fc1ea7a876d476ce0b083d: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-14T12:54:33,319 DEBUG [RS_LOG_REPLAY_OPS-regionserver/669ef94e8b26:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for a572ae08c8738aa198c40f32aa4b5f96: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-14T12:54:33,320 DEBUG [RS_LOG_REPLAY_OPS-regionserver/669ef94e8b26:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for c8ddc4f331afb3a3007ef5c49cdf6ab8: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-14T12:54:33,320 DEBUG [RS_LOG_REPLAY_OPS-regionserver/669ef94e8b26:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/WALs/669ef94e8b26,44819,1731588865173-splitting/669ef94e8b26%2C44819%2C1731588865173.1731588867078 so closing down 2024-11-14T12:54:33,321 DEBUG [RS_LOG_REPLAY_OPS-regionserver/669ef94e8b26:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T12:54:33,321 INFO [RS_LOG_REPLAY_OPS-regionserver/669ef94e8b26:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T12:54:33,321 INFO [RS_LOG_REPLAY_OPS-regionserver/669ef94e8b26:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(425): Processed 10 edits across 0 Regions in 19 ms; skipped=10; WAL=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/WALs/669ef94e8b26,44819,1731588865173-splitting/669ef94e8b26%2C44819%2C1731588865173.1731588867078, size=2.7 K, length=2769, corrupted=false, cancelled=false 2024-11-14T12:54:33,321 DEBUG [RS_LOG_REPLAY_OPS-regionserver/669ef94e8b26:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(428): Completed split of hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/WALs/669ef94e8b26,44819,1731588865173-splitting/669ef94e8b26%2C44819%2C1731588865173.1731588867078, journal: Splitting hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/WALs/669ef94e8b26,44819,1731588865173-splitting/669ef94e8b26%2C44819%2C1731588865173.1731588867078, size=2.7 K (2769bytes) at 1731588873292Finishing writing output for hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/WALs/669ef94e8b26,44819,1731588865173-splitting/669ef94e8b26%2C44819%2C1731588865173.1731588867078 so closing down at 1731588873320 (+28 ms)3 split writer threads finished at 1731588873321 (+1 ms)Processed 10 edits across 0 Regions in 19 ms; skipped=10; WAL=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/WALs/669ef94e8b26,44819,1731588865173-splitting/669ef94e8b26%2C44819%2C1731588865173.1731588867078, size=2.7 K, length=2769, corrupted=false, cancelled=false at 1731588873321 2024-11-14T12:54:33,321 DEBUG [RS_LOG_REPLAY_OPS-regionserver/669ef94e8b26:0-1 {event_type=RS_LOG_REPLAY, pid=66}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/WALs/669ef94e8b26,44819,1731588865173-splitting/669ef94e8b26%2C44819%2C1731588865173.1731588867078 2024-11-14T12:54:33,321 DEBUG [RS_LOG_REPLAY_OPS-regionserver/669ef94e8b26:0-1 {event_type=RS_LOG_REPLAY, pid=66}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=66 2024-11-14T12:54:33,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45309 {}] master.HMaster(4169): Remote procedure done, pid=66 2024-11-14T12:54:33,325 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/WALs/669ef94e8b26,44819,1731588865173-splitting/669ef94e8b26%2C44819%2C1731588865173.1731588867078 to hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/oldWALs 2024-11-14T12:54:33,328 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=66, resume processing ppid=65 2024-11-14T12:54:33,328 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 in 207 msec 2024-11-14T12:54:33,329 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:33,332 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=60 2024-11-14T12:54:33,332 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 in 216 msec 2024-11-14T12:54:33,334 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/WALs/669ef94e8b26,44819,1731588865173-splitting dir is empty, no logs to split. 2024-11-14T12:54:33,334 INFO [PEWorker-1 {}] master.SplitWALManager(105): 669ef94e8b26,44819,1731588865173 WAL count=0, meta=false 2024-11-14T12:54:33,334 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 669ef94e8b26,44819,1731588865173 WAL splitting is done? wals=0, meta=false 2024-11-14T12:54:33,336 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN}, {pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN}, {pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN}, {pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4f16209fa220cf1b7a18b891e4fe53c6, ASSIGN}, {pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=22b9f10374fc1ea7a876d476ce0b083d, ASSIGN}, {pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=18ddd925c95b1b23296976da04f97e83, ASSIGN}, {pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=333a3eba2bb2d318c041beb4642f5ef3, ASSIGN}, {pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2201f29440b3aa3259741bbb374e209d, ASSIGN}] 2024-11-14T12:54:33,338 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2201f29440b3aa3259741bbb374e209d, ASSIGN 2024-11-14T12:54:33,338 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=2201f29440b3aa3259741bbb374e209d, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-14T12:54:33,339 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=333a3eba2bb2d318c041beb4642f5ef3, ASSIGN 2024-11-14T12:54:33,340 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=333a3eba2bb2d318c041beb4642f5ef3, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-14T12:54:33,341 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=18ddd925c95b1b23296976da04f97e83, ASSIGN 2024-11-14T12:54:33,341 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=18ddd925c95b1b23296976da04f97e83, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-14T12:54:33,342 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=22b9f10374fc1ea7a876d476ce0b083d, ASSIGN 2024-11-14T12:54:33,342 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=22b9f10374fc1ea7a876d476ce0b083d, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-14T12:54:33,343 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4f16209fa220cf1b7a18b891e4fe53c6, ASSIGN 2024-11-14T12:54:33,344 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4f16209fa220cf1b7a18b891e4fe53c6, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-14T12:54:33,345 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN 2024-11-14T12:54:33,345 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-14T12:54:33,346 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN 2024-11-14T12:54:33,347 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-14T12:54:33,347 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN 2024-11-14T12:54:33,348 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-14T12:54:33,489 DEBUG [669ef94e8b26:45309 {}] balancer.BalancerClusterState(204): Hosts are {669ef94e8b26=0} racks are {/default-rack=0} 2024-11-14T12:54:33,490 DEBUG [669ef94e8b26:45309 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-14T12:54:33,490 DEBUG [669ef94e8b26:45309 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-14T12:54:33,490 DEBUG [669ef94e8b26:45309 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-14T12:54:33,490 DEBUG [669ef94e8b26:45309 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-14T12:54:33,490 INFO [669ef94e8b26:45309 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-14T12:54:33,490 INFO [669ef94e8b26:45309 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-14T12:54:33,490 DEBUG [669ef94e8b26:45309 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-14T12:54:33,491 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=74 updating hbase:meta row=2201f29440b3aa3259741bbb374e209d, regionState=OPENING, regionLocation=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:33,491 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=c8ddc4f331afb3a3007ef5c49cdf6ab8, regionState=OPENING, regionLocation=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:33,492 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=4f16209fa220cf1b7a18b891e4fe53c6, regionState=OPENING, regionLocation=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:33,493 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 669ef94e8b26:44819 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 669ef94e8b26/172.17.0.3:44819 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T12:54:33,493 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=cdc4b47ed9fe07b56101f9e0e17abcd7, regionState=OPENING, regionLocation=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:33,495 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=22b9f10374fc1ea7a876d476ce0b083d, regionState=OPENING, regionLocation=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:33,496 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=72 updating hbase:meta row=18ddd925c95b1b23296976da04f97e83, regionState=OPENING, regionLocation=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:33,496 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=669ef94e8b26,44819,1731588865173, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=669ef94e8b26,44819,1731588865173, seqNum=-1, error=java.net.ConnectException: Call to address=669ef94e8b26:44819 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 669ef94e8b26/172.17.0.3:44819 2024-11-14T12:54:33,497 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=73 updating hbase:meta row=333a3eba2bb2d318c041beb4642f5ef3, regionState=OPENING, regionLocation=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:33,498 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=669ef94e8b26,44819,1731588865173, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-14T12:54:33,498 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=669ef94e8b26,44819,1731588865173, seqNum=-1 from cache 2024-11-14T12:54:33,498 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=a572ae08c8738aa198c40f32aa4b5f96, regionState=OPENING, regionLocation=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:33,498 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T12:54:33,499 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=669ef94e8b26,43475,1731588864980, seqNum=-1] 2024-11-14T12:54:33,499 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T12:54:33,500 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address 669ef94e8b26:44819 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 669ef94e8b26/172.17.0.3:44819 2024-11-14T12:54:33,501 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 669ef94e8b26:44819 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 669ef94e8b26/172.17.0.3:44819 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T12:54:33,502 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=669ef94e8b26,44819,1731588865173, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=669ef94e8b26,43475,1731588864980, seqNum=-1, error=java.net.ConnectException: Call to address=669ef94e8b26:44819 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 669ef94e8b26/172.17.0.3:44819 2024-11-14T12:54:33,502 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48537, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T12:54:33,502 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=669ef94e8b26,44819,1731588865173, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=669ef94e8b26,43475,1731588864980, seqNum=-1, error=java.net.ConnectException: Call to address=669ef94e8b26:44819 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 669ef94e8b26/172.17.0.3:44819 2024-11-14T12:54:33,502 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=669ef94e8b26,44819,1731588865173, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=669ef94e8b26,43475,1731588864980, seqNum=-1, error=java.net.ConnectException: Call to address=669ef94e8b26:44819 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 669ef94e8b26/172.17.0.3:44819 2024-11-14T12:54:33,503 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=669ef94e8b26,44819,1731588865173, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=669ef94e8b26,43475,1731588864980, seqNum=-1, error=java.net.ConnectException: Call to address=669ef94e8b26:44819 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 669ef94e8b26/172.17.0.3:44819 2024-11-14T12:54:33,503 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=669ef94e8b26,44819,1731588865173, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=669ef94e8b26,43475,1731588864980, seqNum=-1, error=java.net.ConnectException: Call to address=669ef94e8b26:44819 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 669ef94e8b26/172.17.0.3:44819 2024-11-14T12:54:33,503 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=669ef94e8b26,44819,1731588865173, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=669ef94e8b26,43475,1731588864980, seqNum=-1, error=java.net.ConnectException: Call to address=669ef94e8b26:44819 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 669ef94e8b26/172.17.0.3:44819 2024-11-14T12:54:33,503 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address 669ef94e8b26:44819 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 669ef94e8b26/172.17.0.3:44819 2024-11-14T12:54:33,504 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN because future has completed 2024-11-14T12:54:33,505 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure a572ae08c8738aa198c40f32aa4b5f96, server=669ef94e8b26,43475,1731588864980}] 2024-11-14T12:54:33,605 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=2201f29440b3aa3259741bbb374e209d, ASSIGN because future has completed 2024-11-14T12:54:33,605 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2201f29440b3aa3259741bbb374e209d, server=669ef94e8b26,46877,1731588865107}] 2024-11-14T12:54:33,614 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=333a3eba2bb2d318c041beb4642f5ef3, ASSIGN because future has completed 2024-11-14T12:54:33,615 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980}] 2024-11-14T12:54:33,616 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN because future has completed 2024-11-14T12:54:33,617 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4f16209fa220cf1b7a18b891e4fe53c6, ASSIGN because future has completed 2024-11-14T12:54:33,618 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN because future has completed 2024-11-14T12:54:33,619 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=22b9f10374fc1ea7a876d476ce0b083d, ASSIGN because future has completed 2024-11-14T12:54:33,620 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 22b9f10374fc1ea7a876d476ce0b083d, server=669ef94e8b26,43475,1731588864980}] 2024-11-14T12:54:33,620 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=18ddd925c95b1b23296976da04f97e83, ASSIGN because future has completed 2024-11-14T12:54:33,623 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 18ddd925c95b1b23296976da04f97e83, server=669ef94e8b26,43475,1731588864980}] 2024-11-14T12:54:33,627 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure cdc4b47ed9fe07b56101f9e0e17abcd7, server=669ef94e8b26,46877,1731588865107}] 2024-11-14T12:54:33,629 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4f16209fa220cf1b7a18b891e4fe53c6, server=669ef94e8b26,46877,1731588865107}] 2024-11-14T12:54:33,632 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure c8ddc4f331afb3a3007ef5c49cdf6ab8, server=669ef94e8b26,46877,1731588865107}] 2024-11-14T12:54:33,661 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(132): Open test,kkk,1731588868017.333a3eba2bb2d318c041beb4642f5ef3. 2024-11-14T12:54:33,662 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7752): Opening region: {ENCODED => 333a3eba2bb2d318c041beb4642f5ef3, NAME => 'test,kkk,1731588868017.333a3eba2bb2d318c041beb4642f5ef3.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-11-14T12:54:33,662 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 333a3eba2bb2d318c041beb4642f5ef3 2024-11-14T12:54:33,662 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(898): Instantiated test,kkk,1731588868017.333a3eba2bb2d318c041beb4642f5ef3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:33,662 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7794): checking encryption for 333a3eba2bb2d318c041beb4642f5ef3 2024-11-14T12:54:33,662 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7797): checking classloading for 333a3eba2bb2d318c041beb4642f5ef3 2024-11-14T12:54:33,664 INFO [StoreOpener-333a3eba2bb2d318c041beb4642f5ef3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 333a3eba2bb2d318c041beb4642f5ef3 2024-11-14T12:54:33,665 INFO [StoreOpener-333a3eba2bb2d318c041beb4642f5ef3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 333a3eba2bb2d318c041beb4642f5ef3 columnFamilyName family 2024-11-14T12:54:33,665 DEBUG [StoreOpener-333a3eba2bb2d318c041beb4642f5ef3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T12:54:33,666 INFO [StoreOpener-333a3eba2bb2d318c041beb4642f5ef3-1 {}] regionserver.HStore(327): Store=333a3eba2bb2d318c041beb4642f5ef3/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T12:54:33,666 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1038): replaying wal for 333a3eba2bb2d318c041beb4642f5ef3 2024-11-14T12:54:33,667 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/333a3eba2bb2d318c041beb4642f5ef3 2024-11-14T12:54:33,668 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/333a3eba2bb2d318c041beb4642f5ef3 2024-11-14T12:54:33,668 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1048): stopping wal replay for 333a3eba2bb2d318c041beb4642f5ef3 2024-11-14T12:54:33,668 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1060): Cleaning up temporary data for 333a3eba2bb2d318c041beb4642f5ef3 2024-11-14T12:54:33,671 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1093): writing seq id for 333a3eba2bb2d318c041beb4642f5ef3 2024-11-14T12:54:33,672 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1114): Opened 333a3eba2bb2d318c041beb4642f5ef3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69303809, jitterRate=0.032707229256629944}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T12:54:33,672 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 333a3eba2bb2d318c041beb4642f5ef3 2024-11-14T12:54:33,672 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1006): Region open journal for 333a3eba2bb2d318c041beb4642f5ef3: Running coprocessor pre-open hook at 1731588873662Writing region info on filesystem at 1731588873663 (+1 ms)Initializing all the Stores at 1731588873664 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731588873664Cleaning up temporary data from old regions at 1731588873669 (+5 ms)Running coprocessor post-open hooks at 1731588873672 (+3 ms)Region opened successfully at 1731588873672 2024-11-14T12:54:33,673 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1731588868017.333a3eba2bb2d318c041beb4642f5ef3., pid=77, masterSystemTime=1731588873657 2024-11-14T12:54:33,675 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2266): Finished post open deploy task for test,kkk,1731588868017.333a3eba2bb2d318c041beb4642f5ef3. 2024-11-14T12:54:33,675 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(153): Opened test,kkk,1731588868017.333a3eba2bb2d318c041beb4642f5ef3. 2024-11-14T12:54:33,675 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(132): Open test,,1731588868017.a572ae08c8738aa198c40f32aa4b5f96. 2024-11-14T12:54:33,676 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7752): Opening region: {ENCODED => a572ae08c8738aa198c40f32aa4b5f96, NAME => 'test,,1731588868017.a572ae08c8738aa198c40f32aa4b5f96.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-14T12:54:33,676 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test a572ae08c8738aa198c40f32aa4b5f96 2024-11-14T12:54:33,676 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(898): Instantiated test,,1731588868017.a572ae08c8738aa198c40f32aa4b5f96.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:33,676 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7794): checking encryption for a572ae08c8738aa198c40f32aa4b5f96 2024-11-14T12:54:33,676 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7797): checking classloading for a572ae08c8738aa198c40f32aa4b5f96 2024-11-14T12:54:33,677 WARN [PEWorker-1 {}] procedure2.ProcedureTestingUtility(201): Set Kill before store update in rollback to: true 2024-11-14T12:54:33,677 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:33,677 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1844): CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980 java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-14T12:54:33,678 INFO [StoreOpener-a572ae08c8738aa198c40f32aa4b5f96-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region a572ae08c8738aa198c40f32aa4b5f96 2024-11-14T12:54:33,679 INFO [StoreOpener-a572ae08c8738aa198c40f32aa4b5f96-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a572ae08c8738aa198c40f32aa4b5f96 columnFamilyName family 2024-11-14T12:54:33,680 DEBUG [StoreOpener-a572ae08c8738aa198c40f32aa4b5f96-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T12:54:33,680 INFO [StoreOpener-a572ae08c8738aa198c40f32aa4b5f96-1 {}] regionserver.HStore(327): Store=a572ae08c8738aa198c40f32aa4b5f96/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T12:54:33,681 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1038): replaying wal for a572ae08c8738aa198c40f32aa4b5f96 2024-11-14T12:54:33,682 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/a572ae08c8738aa198c40f32aa4b5f96 2024-11-14T12:54:33,684 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/a572ae08c8738aa198c40f32aa4b5f96 2024-11-14T12:54:33,684 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1048): stopping wal replay for a572ae08c8738aa198c40f32aa4b5f96 2024-11-14T12:54:33,684 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1060): Cleaning up temporary data for a572ae08c8738aa198c40f32aa4b5f96 2024-11-14T12:54:33,687 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): writing seq id for a572ae08c8738aa198c40f32aa4b5f96 2024-11-14T12:54:33,686 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=true, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1845) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 4 more 2024-11-14T12:54:33,688 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-14T12:54:33,688 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=82, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure c8ddc4f331afb3a3007ef5c49cdf6ab8, server=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:33,688 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T12:54:33,688 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1114): Opened a572ae08c8738aa198c40f32aa4b5f96; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69873531, jitterRate=0.04119674861431122}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T12:54:33,688 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a572ae08c8738aa198c40f32aa4b5f96 2024-11-14T12:54:33,688 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1006): Region open journal for a572ae08c8738aa198c40f32aa4b5f96: Running coprocessor pre-open hook at 1731588873676Writing region info on filesystem at 1731588873676Initializing all the Stores at 1731588873678 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731588873678Cleaning up temporary data from old regions at 1731588873684 (+6 ms)Running coprocessor post-open hooks at 1731588873688 (+4 ms)Region opened successfully at 1731588873688 2024-11-14T12:54:33,688 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=82, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure c8ddc4f331afb3a3007ef5c49cdf6ab8, server=669ef94e8b26,46877,1731588865107 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-14T12:54:33,689 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1731588868017.a572ae08c8738aa198c40f32aa4b5f96., pid=75, masterSystemTime=1731588873657 2024-11-14T12:54:33,692 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2266): Finished post open deploy task for test,,1731588868017.a572ae08c8738aa198c40f32aa4b5f96. 2024-11-14T12:54:33,692 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(153): Opened test,,1731588868017.a572ae08c8738aa198c40f32aa4b5f96. 2024-11-14T12:54:33,692 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(132): Open test,www,1731588868017.22b9f10374fc1ea7a876d476ce0b083d. 2024-11-14T12:54:33,692 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7752): Opening region: {ENCODED => 22b9f10374fc1ea7a876d476ce0b083d, NAME => 'test,www,1731588868017.22b9f10374fc1ea7a876d476ce0b083d.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-11-14T12:54:33,693 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 22b9f10374fc1ea7a876d476ce0b083d 2024-11-14T12:54:33,693 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(898): Instantiated test,www,1731588868017.22b9f10374fc1ea7a876d476ce0b083d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:33,693 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7794): checking encryption for 22b9f10374fc1ea7a876d476ce0b083d 2024-11-14T12:54:33,693 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7797): checking classloading for 22b9f10374fc1ea7a876d476ce0b083d 2024-11-14T12:54:33,695 INFO [StoreOpener-22b9f10374fc1ea7a876d476ce0b083d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 22b9f10374fc1ea7a876d476ce0b083d 2024-11-14T12:54:33,696 INFO [StoreOpener-22b9f10374fc1ea7a876d476ce0b083d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 22b9f10374fc1ea7a876d476ce0b083d columnFamilyName family 2024-11-14T12:54:33,696 DEBUG [StoreOpener-22b9f10374fc1ea7a876d476ce0b083d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T12:54:33,696 INFO [StoreOpener-22b9f10374fc1ea7a876d476ce0b083d-1 {}] regionserver.HStore(327): Store=22b9f10374fc1ea7a876d476ce0b083d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T12:54:33,696 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1038): replaying wal for 22b9f10374fc1ea7a876d476ce0b083d 2024-11-14T12:54:33,697 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/22b9f10374fc1ea7a876d476ce0b083d 2024-11-14T12:54:33,699 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/22b9f10374fc1ea7a876d476ce0b083d 2024-11-14T12:54:33,699 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1048): stopping wal replay for 22b9f10374fc1ea7a876d476ce0b083d 2024-11-14T12:54:33,700 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1060): Cleaning up temporary data for 22b9f10374fc1ea7a876d476ce0b083d 2024-11-14T12:54:33,702 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1093): writing seq id for 22b9f10374fc1ea7a876d476ce0b083d 2024-11-14T12:54:33,703 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1114): Opened 22b9f10374fc1ea7a876d476ce0b083d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68193222, jitterRate=0.016158193349838257}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T12:54:33,703 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 22b9f10374fc1ea7a876d476ce0b083d 2024-11-14T12:54:33,703 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1006): Region open journal for 22b9f10374fc1ea7a876d476ce0b083d: Running coprocessor pre-open hook at 1731588873693Writing region info on filesystem at 1731588873693Initializing all the Stores at 1731588873694 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731588873694Cleaning up temporary data from old regions at 1731588873700 (+6 ms)Running coprocessor post-open hooks at 1731588873703 (+3 ms)Region opened successfully at 1731588873703 2024-11-14T12:54:33,704 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2236): Post open deploy tasks for test,www,1731588868017.22b9f10374fc1ea7a876d476ce0b083d., pid=78, masterSystemTime=1731588873657 2024-11-14T12:54:33,706 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-14T12:54:33,706 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T12:54:33,706 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2266): Finished post open deploy task for test,www,1731588868017.22b9f10374fc1ea7a876d476ce0b083d. 2024-11-14T12:54:33,706 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T12:54:33,706 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(153): Opened test,www,1731588868017.22b9f10374fc1ea7a876d476ce0b083d. 2024-11-14T12:54:33,707 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open test,qqq,1731588868017.18ddd925c95b1b23296976da04f97e83. 2024-11-14T12:54:33,707 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T12:54:33,707 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => 18ddd925c95b1b23296976da04f97e83, NAME => 'test,qqq,1731588868017.18ddd925c95b1b23296976da04f97e83.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-11-14T12:54:33,707 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 18ddd925c95b1b23296976da04f97e83 2024-11-14T12:54:33,707 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated test,qqq,1731588868017.18ddd925c95b1b23296976da04f97e83.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:33,708 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for 18ddd925c95b1b23296976da04f97e83 2024-11-14T12:54:33,708 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for 18ddd925c95b1b23296976da04f97e83 2024-11-14T12:54:33,708 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-14T12:54:33,708 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-14T12:54:33,708 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-14T12:54:33,708 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45309 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T12:54:33,708 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T12:54:33,709 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-14T12:54:33,709 INFO [StoreOpener-18ddd925c95b1b23296976da04f97e83-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 18ddd925c95b1b23296976da04f97e83 2024-11-14T12:54:33,711 INFO [StoreOpener-18ddd925c95b1b23296976da04f97e83-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 18ddd925c95b1b23296976da04f97e83 columnFamilyName family 2024-11-14T12:54:33,711 DEBUG [StoreOpener-18ddd925c95b1b23296976da04f97e83-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T12:54:33,715 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-14T12:54:33,715 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T12:54:33,715 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T12:54:33,715 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:33,715 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T12:54:33,716 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:33,716 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T12:54:33,716 INFO [StoreOpener-18ddd925c95b1b23296976da04f97e83-1 {}] regionserver.HStore(327): Store=18ddd925c95b1b23296976da04f97e83/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T12:54:33,716 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:33,716 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=82, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T12:54:33,716 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:33,716 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for 18ddd925c95b1b23296976da04f97e83 2024-11-14T12:54:33,716 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=81, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T12:54:33,716 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:33,716 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T12:54:33,716 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:33,716 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T12:54:33,716 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:33,716 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T12:54:33,716 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:33,716 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T12:54:33,716 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T12:54:33,716 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T12:54:33,717 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:33,717 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/18ddd925c95b1b23296976da04f97e83 2024-11-14T12:54:33,717 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T12:54:33,717 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T12:54:33,718 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/18ddd925c95b1b23296976da04f97e83 2024-11-14T12:54:33,718 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for 18ddd925c95b1b23296976da04f97e83 2024-11-14T12:54:33,718 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for 18ddd925c95b1b23296976da04f97e83 2024-11-14T12:54:33,719 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-14T12:54:33,720 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=23f7fbc112f1353900c1181e3ed36707, REOPEN/MOVE 2024-11-14T12:54:33,720 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for 18ddd925c95b1b23296976da04f97e83 2024-11-14T12:54:33,721 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened 18ddd925c95b1b23296976da04f97e83; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63268999, jitterRate=-0.05721844732761383}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T12:54:33,721 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 18ddd925c95b1b23296976da04f97e83 2024-11-14T12:54:33,721 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for 18ddd925c95b1b23296976da04f97e83: Running coprocessor pre-open hook at 1731588873708Writing region info on filesystem at 1731588873708Initializing all the Stores at 1731588873709 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731588873709Cleaning up temporary data from old regions at 1731588873718 (+9 ms)Running coprocessor post-open hooks at 1731588873721 (+3 ms)Region opened successfully at 1731588873721 2024-11-14T12:54:33,722 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2236): Post open deploy tasks for test,qqq,1731588868017.18ddd925c95b1b23296976da04f97e83., pid=79, masterSystemTime=1731588873657 2024-11-14T12:54:33,723 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45309 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T12:54:33,725 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true 2024-11-14T12:54:33,725 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:33,725 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:33,725 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T12:54:33,725 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:33,725 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:33,725 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:33,725 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN 2024-11-14T12:54:33,726 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN 2024-11-14T12:54:33,726 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN 2024-11-14T12:54:33,726 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f16209fa220cf1b7a18b891e4fe53c6, ASSIGN 2024-11-14T12:54:33,726 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22b9f10374fc1ea7a876d476ce0b083d, ASSIGN 2024-11-14T12:54:33,726 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=18ddd925c95b1b23296976da04f97e83, ASSIGN 2024-11-14T12:54:33,726 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=333a3eba2bb2d318c041beb4642f5ef3, ASSIGN 2024-11-14T12:54:33,726 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2201f29440b3aa3259741bbb374e209d, ASSIGN 2024-11-14T12:54:33,726 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure a572ae08c8738aa198c40f32aa4b5f96, server=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:33,726 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2201f29440b3aa3259741bbb374e209d, server=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:33,726 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=73, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:33,726 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 22b9f10374fc1ea7a876d476ce0b083d, server=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:33,726 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 18ddd925c95b1b23296976da04f97e83, server=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:33,726 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure cdc4b47ed9fe07b56101f9e0e17abcd7, server=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:33,726 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=81, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4f16209fa220cf1b7a18b891e4fe53c6, server=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:33,726 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=82, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure c8ddc4f331afb3a3007ef5c49cdf6ab8, server=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:33,727 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:33,727 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,727 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,727 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,727 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=669ef94e8b26,43475,1731588864980 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,727 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,727 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,727 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:33,727 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN 2024-11-14T12:54:33,727 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:33,727 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN 2024-11-14T12:54:33,727 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:33,727 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN 2024-11-14T12:54:33,727 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f16209fa220cf1b7a18b891e4fe53c6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:33,727 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f16209fa220cf1b7a18b891e4fe53c6, ASSIGN 2024-11-14T12:54:33,727 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22b9f10374fc1ea7a876d476ce0b083d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:33,728 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22b9f10374fc1ea7a876d476ce0b083d, ASSIGN 2024-11-14T12:54:33,728 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=18ddd925c95b1b23296976da04f97e83, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:33,728 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=18ddd925c95b1b23296976da04f97e83, ASSIGN 2024-11-14T12:54:33,728 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=333a3eba2bb2d318c041beb4642f5ef3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:33,728 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=333a3eba2bb2d318c041beb4642f5ef3, ASSIGN 2024-11-14T12:54:33,728 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2201f29440b3aa3259741bbb374e209d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:33,728 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2201f29440b3aa3259741bbb374e209d, ASSIGN 2024-11-14T12:54:33,728 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure a572ae08c8738aa198c40f32aa4b5f96, server=669ef94e8b26,43475,1731588864980 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,728 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2201f29440b3aa3259741bbb374e209d, server=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,728 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=73, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,728 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 22b9f10374fc1ea7a876d476ce0b083d, server=669ef94e8b26,43475,1731588864980 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,728 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 18ddd925c95b1b23296976da04f97e83, server=669ef94e8b26,43475,1731588864980 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,728 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure cdc4b47ed9fe07b56101f9e0e17abcd7, server=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,728 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=81, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4f16209fa220cf1b7a18b891e4fe53c6, server=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,728 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=82, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure c8ddc4f331afb3a3007ef5c49cdf6ab8, server=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,729 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 20 msec 2024-11-14T12:54:33,731 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=65 2024-11-14T12:54:33,731 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=669ef94e8b26,43475,1731588864980, table=hbase:meta, region=1588230740 2024-11-14T12:54:33,731 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 669ef94e8b26,43475,1731588864980, state=OPEN 2024-11-14T12:54:33,731 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "669ef94e8b26" port: 43475 start_code: 1731588864980 } transition { transition_code: OPENED region_info { region_id: 1731588868017 table_name { namespace: "default" qualifier: "test" } start_key: "qqq" end_key: "rrr" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 79 initiating_master_active_time: 1731588865297 }; retry (#0) after 1005ms delay (Master is coming online...). org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-14T12:54:33,755 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46877-0x1013ab2e72b0002, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:33,755 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45309-0x1013ab2e72b0000, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:33,755 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43475-0x1013ab2e72b0001, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:33,755 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:33,755 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:33,755 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:33,757 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN to state=OFFLINE, location=null, table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7 to restore RIT 2024-11-14T12:54:33,757 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN to state=OFFLINE, location=null, table=test, region=a572ae08c8738aa198c40f32aa4b5f96 to restore RIT 2024-11-14T12:54:33,757 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN to state=OFFLINE, location=null, table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8 to restore RIT 2024-11-14T12:54:33,757 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f16209fa220cf1b7a18b891e4fe53c6, ASSIGN to state=OFFLINE, location=null, table=test, region=4f16209fa220cf1b7a18b891e4fe53c6 to restore RIT 2024-11-14T12:54:33,757 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22b9f10374fc1ea7a876d476ce0b083d, ASSIGN to state=OFFLINE, location=null, table=test, region=22b9f10374fc1ea7a876d476ce0b083d to restore RIT 2024-11-14T12:54:33,757 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=18ddd925c95b1b23296976da04f97e83, ASSIGN to state=OFFLINE, location=null, table=test, region=18ddd925c95b1b23296976da04f97e83 to restore RIT 2024-11-14T12:54:33,757 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=333a3eba2bb2d318c041beb4642f5ef3, ASSIGN to state=OFFLINE, location=null, table=test, region=333a3eba2bb2d318c041beb4642f5ef3 to restore RIT 2024-11-14T12:54:33,757 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2201f29440b3aa3259741bbb374e209d, ASSIGN to state=OFFLINE, location=null, table=test, region=2201f29440b3aa3259741bbb374e209d to restore RIT 2024-11-14T12:54:33,758 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731588903758 2024-11-14T12:54:33,758 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-14T12:54:33,758 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-14T12:54:33,760 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-14T12:54:33,762 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-14T12:54:33,762 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=81, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4f16209fa220cf1b7a18b891e4fe53c6, server=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:33,762 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T12:54:33,762 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=81, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4f16209fa220cf1b7a18b891e4fe53c6, server=669ef94e8b26,46877,1731588865107 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-14T12:54:33,764 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(132): Open test,zzz,1731588868017.2201f29440b3aa3259741bbb374e209d. 2024-11-14T12:54:33,765 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7752): Opening region: {ENCODED => 2201f29440b3aa3259741bbb374e209d, NAME => 'test,zzz,1731588868017.2201f29440b3aa3259741bbb374e209d.', STARTKEY => 'zzz', ENDKEY => ''} 2024-11-14T12:54:33,765 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 2201f29440b3aa3259741bbb374e209d 2024-11-14T12:54:33,765 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(898): Instantiated test,zzz,1731588868017.2201f29440b3aa3259741bbb374e209d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:33,765 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7794): checking encryption for 2201f29440b3aa3259741bbb374e209d 2024-11-14T12:54:33,765 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7797): checking classloading for 2201f29440b3aa3259741bbb374e209d 2024-11-14T12:54:33,767 INFO [StoreOpener-2201f29440b3aa3259741bbb374e209d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 2201f29440b3aa3259741bbb374e209d 2024-11-14T12:54:33,768 INFO [StoreOpener-2201f29440b3aa3259741bbb374e209d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2201f29440b3aa3259741bbb374e209d columnFamilyName family 2024-11-14T12:54:33,768 DEBUG [StoreOpener-2201f29440b3aa3259741bbb374e209d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T12:54:33,769 INFO [StoreOpener-2201f29440b3aa3259741bbb374e209d-1 {}] regionserver.HStore(327): Store=2201f29440b3aa3259741bbb374e209d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T12:54:33,769 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1038): replaying wal for 2201f29440b3aa3259741bbb374e209d 2024-11-14T12:54:33,770 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/2201f29440b3aa3259741bbb374e209d 2024-11-14T12:54:33,771 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/2201f29440b3aa3259741bbb374e209d 2024-11-14T12:54:33,772 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1048): stopping wal replay for 2201f29440b3aa3259741bbb374e209d 2024-11-14T12:54:33,772 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1060): Cleaning up temporary data for 2201f29440b3aa3259741bbb374e209d 2024-11-14T12:54:33,774 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1093): writing seq id for 2201f29440b3aa3259741bbb374e209d 2024-11-14T12:54:33,775 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1114): Opened 2201f29440b3aa3259741bbb374e209d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66173308, jitterRate=-0.013940870761871338}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T12:54:33,775 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2201f29440b3aa3259741bbb374e209d 2024-11-14T12:54:33,775 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1006): Region open journal for 2201f29440b3aa3259741bbb374e209d: Running coprocessor pre-open hook at 1731588873765Writing region info on filesystem at 1731588873765Initializing all the Stores at 1731588873767 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731588873767Cleaning up temporary data from old regions at 1731588873772 (+5 ms)Running coprocessor post-open hooks at 1731588873775 (+3 ms)Region opened successfully at 1731588873775 2024-11-14T12:54:33,776 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2236): Post open deploy tasks for test,zzz,1731588868017.2201f29440b3aa3259741bbb374e209d., pid=76, masterSystemTime=1731588873757 2024-11-14T12:54:33,777 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45309 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.PleaseHoldException: Master not fully online; hbase:meta=false, metaLoaded=false at org.apache.hadoop.hbase.master.assignment.AssignmentManager.checkMetaLoaded(AssignmentManager.java:1916) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.updateRegionTransition(AssignmentManager.java:1280) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:1206) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:1256) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1867) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T12:54:33,781 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45309 {}] ipc.CallRunner(138): callId: 86 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 123 connection: 172.17.0.3:36833 deadline: 1731588883777, exception=org.apache.hadoop.hbase.PleaseHoldException: Master not fully online; hbase:meta=false, metaLoaded=false 2024-11-14T12:54:33,783 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "669ef94e8b26" port: 46877 start_code: 1731588865107 } transition { transition_code: OPENED region_info { region_id: 1731588868017 table_name { namespace: "default" qualifier: "test" } start_key: "zzz" end_key: "" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 76 initiating_master_active_time: 1731588865297 }; retry (#0) after 1008ms delay (Master is coming online...). org.apache.hadoop.hbase.PleaseHoldException: org.apache.hadoop.hbase.PleaseHoldException: Master not fully online; hbase:meta=false, metaLoaded=false at org.apache.hadoop.hbase.master.assignment.AssignmentManager.checkMetaLoaded(AssignmentManager.java:1916) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.updateRegionTransition(AssignmentManager.java:1280) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:1206) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:1256) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1867) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.PleaseHoldException: Master not fully online; hbase:meta=false, metaLoaded=false at org.apache.hadoop.hbase.master.assignment.AssignmentManager.checkMetaLoaded(AssignmentManager.java:1916) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.updateRegionTransition(AssignmentManager.java:1280) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:1206) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:1256) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1867) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-14T12:54:33,791 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a572ae08c8738aa198c40f32aa4b5f96, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:33,791 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2ccfc8c17ab1571bc4ea6d526017f9f, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:33,791 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd0512a8a2f36befd3453e1140df3af, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:33,792 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b8044551b940cf51bfa7c7633c4df21d, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:33,792 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f7fbc112f1353900c1181e3ed36707, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=5 2024-11-14T12:54:33,792 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf47be17fd22bc204db28da07b30af2c, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:33,793 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f76ec25f51f69b1106342a7dfae65fb6, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:33,793 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cdc4b47ed9fe07b56101f9e0e17abcd7, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:33,793 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03f289a2805f0b2f037581e9390ba191, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:33,793 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1358fa84af33d1799bd11ba7f4959a37, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:33,793 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=333a3eba2bb2d318c041beb4642f5ef3, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:33,794 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64b5bd4e6112417a55e871b36551c5b8, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:33,794 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e90cc230c4f13fece446e3e8420dddbe, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:33,794 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f16209fa220cf1b7a18b891e4fe53c6, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:33,794 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=338b47e35e9a8b9dd3045d365aaf47bd, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:33,795 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ea6e95809b33f853859fa227179bfe85, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:33,795 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=18ddd925c95b1b23296976da04f97e83, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:33,795 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5d884ba0737bc572475e022a6ed1bc8, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:33,796 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d33b9e9c25fcfc654d257646677381c, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:33,796 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8ddc4f331afb3a3007ef5c49cdf6ab8, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:33,796 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8c0d62856832eec9f9ca20bb6db602c, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:33,797 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87eb074dbd46b3bc6e27fb0bf3291d5b, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:33,797 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22b9f10374fc1ea7a876d476ce0b083d, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:33,798 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cadcf21958eaec53ee489ca0918e3ab7, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:33,798 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66b34b9be40614bd68d0cea995e44193, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:33,798 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2201f29440b3aa3259741bbb374e209d, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:33,798 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-14T12:54:33,798 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731588933798 2024-11-14T12:54:33,799 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731588993798 2024-11-14T12:54:33,799 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 40 msec 2024-11-14T12:54:33,799 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-14T12:54:33,799 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T12:54:33,799 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T12:54:33,799 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T12:54:33,799 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-14T12:54:33,800 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-14T12:54:33,800 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-14T12:54:33,800 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T12:54:33,801 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-14T12:54:33,806 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-14T12:54:33,806 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T12:54:33,806 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T12:54:33,806 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:33,806 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T12:54:33,806 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:33,806 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T12:54:33,807 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:33,807 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:33,807 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=81, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T12:54:33,807 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:33,807 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T12:54:33,807 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:33,807 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T12:54:33,807 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:33,807 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T12:54:33,807 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:33,807 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T12:54:33,807 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T12:54:33,807 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T12:54:33,807 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:33,807 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T12:54:33,807 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T12:54:33,809 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-14T12:54:33,809 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=23f7fbc112f1353900c1181e3ed36707, REOPEN/MOVE 2024-11-14T12:54:33,809 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45309 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T12:54:33,813 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true 2024-11-14T12:54:33,814 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:33,814 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:33,814 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T12:54:33,814 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:33,814 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:33,814 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:33,814 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN 2024-11-14T12:54:33,814 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN 2024-11-14T12:54:33,814 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN 2024-11-14T12:54:33,814 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f16209fa220cf1b7a18b891e4fe53c6, ASSIGN 2024-11-14T12:54:33,814 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22b9f10374fc1ea7a876d476ce0b083d, ASSIGN 2024-11-14T12:54:33,814 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=18ddd925c95b1b23296976da04f97e83, ASSIGN 2024-11-14T12:54:33,814 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=333a3eba2bb2d318c041beb4642f5ef3, ASSIGN 2024-11-14T12:54:33,814 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2201f29440b3aa3259741bbb374e209d, ASSIGN 2024-11-14T12:54:33,814 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure a572ae08c8738aa198c40f32aa4b5f96, server=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:33,814 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2201f29440b3aa3259741bbb374e209d, server=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:33,815 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=73, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:33,815 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 22b9f10374fc1ea7a876d476ce0b083d, server=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:33,815 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 18ddd925c95b1b23296976da04f97e83, server=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:33,815 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure cdc4b47ed9fe07b56101f9e0e17abcd7, server=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:33,815 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=81, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4f16209fa220cf1b7a18b891e4fe53c6, server=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:33,815 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:33,815 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,815 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,815 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,815 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=669ef94e8b26,43475,1731588864980 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,815 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,815 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,815 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:33,815 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN 2024-11-14T12:54:33,815 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:33,816 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN 2024-11-14T12:54:33,816 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:33,816 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN 2024-11-14T12:54:33,816 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f16209fa220cf1b7a18b891e4fe53c6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:33,816 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f16209fa220cf1b7a18b891e4fe53c6, ASSIGN 2024-11-14T12:54:33,816 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22b9f10374fc1ea7a876d476ce0b083d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:33,816 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22b9f10374fc1ea7a876d476ce0b083d, ASSIGN 2024-11-14T12:54:33,816 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=18ddd925c95b1b23296976da04f97e83, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:33,816 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=18ddd925c95b1b23296976da04f97e83, ASSIGN 2024-11-14T12:54:33,816 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=333a3eba2bb2d318c041beb4642f5ef3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:33,816 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=333a3eba2bb2d318c041beb4642f5ef3, ASSIGN 2024-11-14T12:54:33,816 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2201f29440b3aa3259741bbb374e209d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:33,816 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2201f29440b3aa3259741bbb374e209d, ASSIGN 2024-11-14T12:54:33,816 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure a572ae08c8738aa198c40f32aa4b5f96, server=669ef94e8b26,43475,1731588864980 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,816 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2201f29440b3aa3259741bbb374e209d, server=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,816 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=73, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,816 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 22b9f10374fc1ea7a876d476ce0b083d, server=669ef94e8b26,43475,1731588864980 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,816 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 18ddd925c95b1b23296976da04f97e83, server=669ef94e8b26,43475,1731588864980 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,816 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure cdc4b47ed9fe07b56101f9e0e17abcd7, server=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,817 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=81, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4f16209fa220cf1b7a18b891e4fe53c6, server=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,817 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 15 msec 2024-11-14T12:54:33,818 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=65 2024-11-14T12:54:33,818 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=669ef94e8b26,43475,1731588864980, table=hbase:meta, region=1588230740 2024-11-14T12:54:33,818 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 669ef94e8b26,43475,1731588864980, state=OPEN 2024-11-14T12:54:33,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43475-0x1013ab2e72b0001, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:33,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45309-0x1013ab2e72b0000, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:33,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46877-0x1013ab2e72b0002, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:33,828 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:33,828 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:33,829 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:33,829 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN to state=OFFLINE, location=null, table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7 to restore RIT 2024-11-14T12:54:33,829 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN to state=OFFLINE, location=null, table=test, region=a572ae08c8738aa198c40f32aa4b5f96 to restore RIT 2024-11-14T12:54:33,829 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN to state=OFFLINE, location=null, table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8 to restore RIT 2024-11-14T12:54:33,829 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f16209fa220cf1b7a18b891e4fe53c6, ASSIGN to state=OFFLINE, location=null, table=test, region=4f16209fa220cf1b7a18b891e4fe53c6 to restore RIT 2024-11-14T12:54:33,829 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22b9f10374fc1ea7a876d476ce0b083d, ASSIGN to state=OFFLINE, location=null, table=test, region=22b9f10374fc1ea7a876d476ce0b083d to restore RIT 2024-11-14T12:54:33,829 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=18ddd925c95b1b23296976da04f97e83, ASSIGN to state=OFFLINE, location=null, table=test, region=18ddd925c95b1b23296976da04f97e83 to restore RIT 2024-11-14T12:54:33,830 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=333a3eba2bb2d318c041beb4642f5ef3, ASSIGN to state=OFFLINE, location=null, table=test, region=333a3eba2bb2d318c041beb4642f5ef3 to restore RIT 2024-11-14T12:54:33,830 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2201f29440b3aa3259741bbb374e209d, ASSIGN to state=OFFLINE, location=null, table=test, region=2201f29440b3aa3259741bbb374e209d to restore RIT 2024-11-14T12:54:33,831 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731588903831 2024-11-14T12:54:33,831 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-14T12:54:33,831 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-14T12:54:33,833 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-14T12:54:33,834 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-14T12:54:33,834 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=80, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure cdc4b47ed9fe07b56101f9e0e17abcd7, server=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:33,834 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T12:54:33,834 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=80, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure cdc4b47ed9fe07b56101f9e0e17abcd7, server=669ef94e8b26,46877,1731588865107 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-14T12:54:33,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a572ae08c8738aa198c40f32aa4b5f96, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:33,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2ccfc8c17ab1571bc4ea6d526017f9f, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:33,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd0512a8a2f36befd3453e1140df3af, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:33,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b8044551b940cf51bfa7c7633c4df21d, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:33,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f7fbc112f1353900c1181e3ed36707, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=5 2024-11-14T12:54:33,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf47be17fd22bc204db28da07b30af2c, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:33,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f76ec25f51f69b1106342a7dfae65fb6, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:33,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cdc4b47ed9fe07b56101f9e0e17abcd7, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:33,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03f289a2805f0b2f037581e9390ba191, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:33,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1358fa84af33d1799bd11ba7f4959a37, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:33,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=333a3eba2bb2d318c041beb4642f5ef3, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:33,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64b5bd4e6112417a55e871b36551c5b8, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:33,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e90cc230c4f13fece446e3e8420dddbe, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:33,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f16209fa220cf1b7a18b891e4fe53c6, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:33,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=338b47e35e9a8b9dd3045d365aaf47bd, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:33,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ea6e95809b33f853859fa227179bfe85, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:33,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=18ddd925c95b1b23296976da04f97e83, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:33,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5d884ba0737bc572475e022a6ed1bc8, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:33,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d33b9e9c25fcfc654d257646677381c, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:33,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8ddc4f331afb3a3007ef5c49cdf6ab8, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:33,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8c0d62856832eec9f9ca20bb6db602c, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:33,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87eb074dbd46b3bc6e27fb0bf3291d5b, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:33,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22b9f10374fc1ea7a876d476ce0b083d, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:33,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cadcf21958eaec53ee489ca0918e3ab7, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:33,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66b34b9be40614bd68d0cea995e44193, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:33,847 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2201f29440b3aa3259741bbb374e209d, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:33,847 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-14T12:54:33,847 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731588933847 2024-11-14T12:54:33,847 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731588993847 2024-11-14T12:54:33,847 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 15 msec 2024-11-14T12:54:33,847 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-14T12:54:33,847 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T12:54:33,847 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T12:54:33,847 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T12:54:33,847 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-14T12:54:33,847 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-14T12:54:33,847 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-14T12:54:33,848 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T12:54:33,850 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 2 msec 2024-11-14T12:54:33,855 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-14T12:54:33,855 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T12:54:33,855 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T12:54:33,855 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:33,855 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T12:54:33,856 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:33,856 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T12:54:33,856 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:33,856 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:33,856 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:33,856 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T12:54:33,856 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:33,856 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T12:54:33,856 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:33,856 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T12:54:33,856 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:33,856 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T12:54:33,856 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T12:54:33,856 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T12:54:33,856 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:33,856 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T12:54:33,856 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T12:54:33,858 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-14T12:54:33,858 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=23f7fbc112f1353900c1181e3ed36707, REOPEN/MOVE 2024-11-14T12:54:33,860 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true 2024-11-14T12:54:33,860 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:33,860 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:33,860 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T12:54:33,860 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:33,860 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:33,860 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:33,860 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN 2024-11-14T12:54:33,860 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN 2024-11-14T12:54:33,860 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN 2024-11-14T12:54:33,860 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f16209fa220cf1b7a18b891e4fe53c6, ASSIGN 2024-11-14T12:54:33,860 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22b9f10374fc1ea7a876d476ce0b083d, ASSIGN 2024-11-14T12:54:33,860 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=18ddd925c95b1b23296976da04f97e83, ASSIGN 2024-11-14T12:54:33,860 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=333a3eba2bb2d318c041beb4642f5ef3, ASSIGN 2024-11-14T12:54:33,861 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2201f29440b3aa3259741bbb374e209d, ASSIGN 2024-11-14T12:54:33,861 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure a572ae08c8738aa198c40f32aa4b5f96, server=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:33,861 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2201f29440b3aa3259741bbb374e209d, server=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:33,861 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=73, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:33,861 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 22b9f10374fc1ea7a876d476ce0b083d, server=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:33,861 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 18ddd925c95b1b23296976da04f97e83, server=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:33,861 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure cdc4b47ed9fe07b56101f9e0e17abcd7, server=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:33,861 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:33,861 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,861 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,861 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,861 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=669ef94e8b26,43475,1731588864980 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,861 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,861 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,861 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:33,861 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN 2024-11-14T12:54:33,861 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:33,861 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN 2024-11-14T12:54:33,861 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:33,861 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN 2024-11-14T12:54:33,861 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f16209fa220cf1b7a18b891e4fe53c6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:33,861 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f16209fa220cf1b7a18b891e4fe53c6, ASSIGN 2024-11-14T12:54:33,861 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22b9f10374fc1ea7a876d476ce0b083d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:33,861 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22b9f10374fc1ea7a876d476ce0b083d, ASSIGN 2024-11-14T12:54:33,861 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=18ddd925c95b1b23296976da04f97e83, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:33,861 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=18ddd925c95b1b23296976da04f97e83, ASSIGN 2024-11-14T12:54:33,861 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=333a3eba2bb2d318c041beb4642f5ef3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:33,862 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=333a3eba2bb2d318c041beb4642f5ef3, ASSIGN 2024-11-14T12:54:33,862 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2201f29440b3aa3259741bbb374e209d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:33,862 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2201f29440b3aa3259741bbb374e209d, ASSIGN 2024-11-14T12:54:33,862 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure a572ae08c8738aa198c40f32aa4b5f96, server=669ef94e8b26,43475,1731588864980 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,862 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2201f29440b3aa3259741bbb374e209d, server=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,862 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=73, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,862 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 22b9f10374fc1ea7a876d476ce0b083d, server=669ef94e8b26,43475,1731588864980 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,862 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 18ddd925c95b1b23296976da04f97e83, server=669ef94e8b26,43475,1731588864980 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,862 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure cdc4b47ed9fe07b56101f9e0e17abcd7, server=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,862 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 11 msec 2024-11-14T12:54:33,863 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=65 2024-11-14T12:54:33,863 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=669ef94e8b26,43475,1731588864980, table=hbase:meta, region=1588230740 2024-11-14T12:54:33,863 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 669ef94e8b26,43475,1731588864980, state=OPEN 2024-11-14T12:54:33,872 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46877-0x1013ab2e72b0002, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:33,872 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45309-0x1013ab2e72b0000, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:33,872 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43475-0x1013ab2e72b0001, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:33,872 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:33,872 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:33,872 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:33,873 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN to state=OFFLINE, location=null, table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7 to restore RIT 2024-11-14T12:54:33,873 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN to state=OFFLINE, location=null, table=test, region=a572ae08c8738aa198c40f32aa4b5f96 to restore RIT 2024-11-14T12:54:33,873 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN to state=OFFLINE, location=null, table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8 to restore RIT 2024-11-14T12:54:33,873 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f16209fa220cf1b7a18b891e4fe53c6, ASSIGN to state=OFFLINE, location=null, table=test, region=4f16209fa220cf1b7a18b891e4fe53c6 to restore RIT 2024-11-14T12:54:33,873 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22b9f10374fc1ea7a876d476ce0b083d, ASSIGN to state=OFFLINE, location=null, table=test, region=22b9f10374fc1ea7a876d476ce0b083d to restore RIT 2024-11-14T12:54:33,873 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=18ddd925c95b1b23296976da04f97e83, ASSIGN to state=OFFLINE, location=null, table=test, region=18ddd925c95b1b23296976da04f97e83 to restore RIT 2024-11-14T12:54:33,874 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=333a3eba2bb2d318c041beb4642f5ef3, ASSIGN to state=OFFLINE, location=null, table=test, region=333a3eba2bb2d318c041beb4642f5ef3 to restore RIT 2024-11-14T12:54:33,874 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2201f29440b3aa3259741bbb374e209d, ASSIGN to state=OFFLINE, location=null, table=test, region=2201f29440b3aa3259741bbb374e209d to restore RIT 2024-11-14T12:54:33,875 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731588903875 2024-11-14T12:54:33,875 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-14T12:54:33,875 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-14T12:54:33,877 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-14T12:54:33,878 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-14T12:54:33,879 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=79, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 18ddd925c95b1b23296976da04f97e83, server=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:33,879 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T12:54:33,879 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=79, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 18ddd925c95b1b23296976da04f97e83, server=669ef94e8b26,43475,1731588864980 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-14T12:54:33,885 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a572ae08c8738aa198c40f32aa4b5f96, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:33,885 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2ccfc8c17ab1571bc4ea6d526017f9f, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:33,885 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd0512a8a2f36befd3453e1140df3af, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:33,886 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b8044551b940cf51bfa7c7633c4df21d, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:33,886 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f7fbc112f1353900c1181e3ed36707, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=5 2024-11-14T12:54:33,886 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf47be17fd22bc204db28da07b30af2c, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:33,886 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f76ec25f51f69b1106342a7dfae65fb6, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:33,886 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cdc4b47ed9fe07b56101f9e0e17abcd7, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:33,887 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03f289a2805f0b2f037581e9390ba191, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:33,887 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1358fa84af33d1799bd11ba7f4959a37, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:33,887 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=333a3eba2bb2d318c041beb4642f5ef3, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:33,887 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64b5bd4e6112417a55e871b36551c5b8, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:33,888 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e90cc230c4f13fece446e3e8420dddbe, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:33,888 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f16209fa220cf1b7a18b891e4fe53c6, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:33,888 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=338b47e35e9a8b9dd3045d365aaf47bd, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:33,888 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ea6e95809b33f853859fa227179bfe85, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:33,888 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=18ddd925c95b1b23296976da04f97e83, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:33,889 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5d884ba0737bc572475e022a6ed1bc8, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:33,889 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d33b9e9c25fcfc654d257646677381c, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:33,889 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8ddc4f331afb3a3007ef5c49cdf6ab8, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:33,889 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8c0d62856832eec9f9ca20bb6db602c, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:33,889 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87eb074dbd46b3bc6e27fb0bf3291d5b, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:33,890 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22b9f10374fc1ea7a876d476ce0b083d, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:33,890 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cadcf21958eaec53ee489ca0918e3ab7, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:33,890 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66b34b9be40614bd68d0cea995e44193, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:33,890 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2201f29440b3aa3259741bbb374e209d, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:33,890 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-14T12:54:33,890 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731588933890 2024-11-14T12:54:33,890 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731588993890 2024-11-14T12:54:33,890 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 15 msec 2024-11-14T12:54:33,890 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-14T12:54:33,890 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T12:54:33,890 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T12:54:33,891 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T12:54:33,891 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-14T12:54:33,891 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-14T12:54:33,891 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-14T12:54:33,891 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T12:54:33,892 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-14T12:54:33,895 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-14T12:54:33,895 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T12:54:33,895 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T12:54:33,895 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:33,895 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:33,895 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T12:54:33,895 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:33,895 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:33,895 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:33,895 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T12:54:33,895 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:33,895 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T12:54:33,895 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:33,895 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T12:54:33,895 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:33,895 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T12:54:33,896 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T12:54:33,896 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T12:54:33,896 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:33,896 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T12:54:33,896 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T12:54:33,897 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-14T12:54:33,897 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=23f7fbc112f1353900c1181e3ed36707, REOPEN/MOVE 2024-11-14T12:54:33,899 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true 2024-11-14T12:54:33,899 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:33,899 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:33,899 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T12:54:33,899 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:33,899 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:33,899 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:33,899 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN 2024-11-14T12:54:33,899 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN 2024-11-14T12:54:33,900 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN 2024-11-14T12:54:33,900 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f16209fa220cf1b7a18b891e4fe53c6, ASSIGN 2024-11-14T12:54:33,900 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22b9f10374fc1ea7a876d476ce0b083d, ASSIGN 2024-11-14T12:54:33,900 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=18ddd925c95b1b23296976da04f97e83, ASSIGN 2024-11-14T12:54:33,900 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=333a3eba2bb2d318c041beb4642f5ef3, ASSIGN 2024-11-14T12:54:33,900 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2201f29440b3aa3259741bbb374e209d, ASSIGN 2024-11-14T12:54:33,900 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure a572ae08c8738aa198c40f32aa4b5f96, server=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:33,900 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2201f29440b3aa3259741bbb374e209d, server=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:33,900 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=73, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:33,900 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 22b9f10374fc1ea7a876d476ce0b083d, server=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:33,900 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 18ddd925c95b1b23296976da04f97e83, server=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:33,900 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:33,900 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,900 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,900 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,900 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=669ef94e8b26,43475,1731588864980 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,900 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,900 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,900 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:33,901 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN 2024-11-14T12:54:33,901 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:33,901 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN 2024-11-14T12:54:33,901 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:33,901 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN 2024-11-14T12:54:33,901 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f16209fa220cf1b7a18b891e4fe53c6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:33,901 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f16209fa220cf1b7a18b891e4fe53c6, ASSIGN 2024-11-14T12:54:33,901 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22b9f10374fc1ea7a876d476ce0b083d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:33,901 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22b9f10374fc1ea7a876d476ce0b083d, ASSIGN 2024-11-14T12:54:33,901 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=18ddd925c95b1b23296976da04f97e83, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:33,901 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=18ddd925c95b1b23296976da04f97e83, ASSIGN 2024-11-14T12:54:33,901 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=333a3eba2bb2d318c041beb4642f5ef3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:33,901 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=333a3eba2bb2d318c041beb4642f5ef3, ASSIGN 2024-11-14T12:54:33,901 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2201f29440b3aa3259741bbb374e209d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:33,901 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2201f29440b3aa3259741bbb374e209d, ASSIGN 2024-11-14T12:54:33,901 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure a572ae08c8738aa198c40f32aa4b5f96, server=669ef94e8b26,43475,1731588864980 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,901 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2201f29440b3aa3259741bbb374e209d, server=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,902 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=73, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,902 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 22b9f10374fc1ea7a876d476ce0b083d, server=669ef94e8b26,43475,1731588864980 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,902 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 18ddd925c95b1b23296976da04f97e83, server=669ef94e8b26,43475,1731588864980 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,902 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-14T12:54:33,903 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=65 2024-11-14T12:54:33,903 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=669ef94e8b26,43475,1731588864980, table=hbase:meta, region=1588230740 2024-11-14T12:54:33,903 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 669ef94e8b26,43475,1731588864980, state=OPEN 2024-11-14T12:54:33,913 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45309 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T12:54:33,913 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45309-0x1013ab2e72b0000, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:33,914 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:33,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46877-0x1013ab2e72b0002, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:33,914 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:33,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43475-0x1013ab2e72b0001, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:33,914 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:33,916 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN to state=OFFLINE, location=null, table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7 to restore RIT 2024-11-14T12:54:33,916 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN to state=OFFLINE, location=null, table=test, region=a572ae08c8738aa198c40f32aa4b5f96 to restore RIT 2024-11-14T12:54:33,916 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN to state=OFFLINE, location=null, table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8 to restore RIT 2024-11-14T12:54:33,916 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f16209fa220cf1b7a18b891e4fe53c6, ASSIGN to state=OFFLINE, location=null, table=test, region=4f16209fa220cf1b7a18b891e4fe53c6 to restore RIT 2024-11-14T12:54:33,916 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22b9f10374fc1ea7a876d476ce0b083d, ASSIGN to state=OFFLINE, location=null, table=test, region=22b9f10374fc1ea7a876d476ce0b083d to restore RIT 2024-11-14T12:54:33,916 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=18ddd925c95b1b23296976da04f97e83, ASSIGN to state=OFFLINE, location=null, table=test, region=18ddd925c95b1b23296976da04f97e83 to restore RIT 2024-11-14T12:54:33,916 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=333a3eba2bb2d318c041beb4642f5ef3, ASSIGN to state=OFFLINE, location=null, table=test, region=333a3eba2bb2d318c041beb4642f5ef3 to restore RIT 2024-11-14T12:54:33,917 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2201f29440b3aa3259741bbb374e209d, ASSIGN to state=OFFLINE, location=null, table=test, region=2201f29440b3aa3259741bbb374e209d to restore RIT 2024-11-14T12:54:33,917 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731588903917 2024-11-14T12:54:33,917 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-14T12:54:33,917 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-14T12:54:33,919 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-14T12:54:33,924 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-14T12:54:33,924 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 22b9f10374fc1ea7a876d476ce0b083d, server=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:33,924 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T12:54:33,924 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 22b9f10374fc1ea7a876d476ce0b083d, server=669ef94e8b26,43475,1731588864980 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-14T12:54:33,924 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a572ae08c8738aa198c40f32aa4b5f96, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:33,924 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2ccfc8c17ab1571bc4ea6d526017f9f, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:33,925 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd0512a8a2f36befd3453e1140df3af, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:33,925 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b8044551b940cf51bfa7c7633c4df21d, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:33,925 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f7fbc112f1353900c1181e3ed36707, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=5 2024-11-14T12:54:33,925 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf47be17fd22bc204db28da07b30af2c, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:33,925 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f76ec25f51f69b1106342a7dfae65fb6, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:33,926 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cdc4b47ed9fe07b56101f9e0e17abcd7, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:33,926 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03f289a2805f0b2f037581e9390ba191, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:33,926 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1358fa84af33d1799bd11ba7f4959a37, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:33,926 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=333a3eba2bb2d318c041beb4642f5ef3, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:33,926 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64b5bd4e6112417a55e871b36551c5b8, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:33,927 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e90cc230c4f13fece446e3e8420dddbe, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:33,927 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f16209fa220cf1b7a18b891e4fe53c6, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:33,927 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=338b47e35e9a8b9dd3045d365aaf47bd, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:33,927 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ea6e95809b33f853859fa227179bfe85, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:33,927 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=18ddd925c95b1b23296976da04f97e83, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:33,928 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5d884ba0737bc572475e022a6ed1bc8, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:33,928 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d33b9e9c25fcfc654d257646677381c, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:33,928 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8ddc4f331afb3a3007ef5c49cdf6ab8, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:33,928 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8c0d62856832eec9f9ca20bb6db602c, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:33,928 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87eb074dbd46b3bc6e27fb0bf3291d5b, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:33,929 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22b9f10374fc1ea7a876d476ce0b083d, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:33,929 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cadcf21958eaec53ee489ca0918e3ab7, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:33,929 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66b34b9be40614bd68d0cea995e44193, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:33,929 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2201f29440b3aa3259741bbb374e209d, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:33,929 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-14T12:54:33,930 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731588933929 2024-11-14T12:54:33,930 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731588993930 2024-11-14T12:54:33,930 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 12 msec 2024-11-14T12:54:33,930 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-14T12:54:33,930 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T12:54:33,930 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T12:54:33,930 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T12:54:33,930 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-14T12:54:33,930 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-14T12:54:33,930 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-14T12:54:33,931 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T12:54:33,932 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-14T12:54:33,935 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-14T12:54:33,936 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T12:54:33,936 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T12:54:33,936 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:33,936 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:33,936 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T12:54:33,936 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:33,936 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:33,936 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45309 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T12:54:33,936 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:33,936 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T12:54:33,936 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:33,936 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:33,936 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T12:54:33,936 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:33,936 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T12:54:33,936 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T12:54:33,936 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T12:54:33,937 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:33,937 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T12:54:33,937 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T12:54:33,938 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-14T12:54:33,938 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=23f7fbc112f1353900c1181e3ed36707, REOPEN/MOVE 2024-11-14T12:54:33,940 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true 2024-11-14T12:54:33,940 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:33,941 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:33,941 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T12:54:33,941 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:33,941 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:33,941 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:33,941 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN 2024-11-14T12:54:33,941 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN 2024-11-14T12:54:33,941 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN 2024-11-14T12:54:33,941 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f16209fa220cf1b7a18b891e4fe53c6, ASSIGN 2024-11-14T12:54:33,941 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22b9f10374fc1ea7a876d476ce0b083d, ASSIGN 2024-11-14T12:54:33,941 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=18ddd925c95b1b23296976da04f97e83, ASSIGN 2024-11-14T12:54:33,941 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=333a3eba2bb2d318c041beb4642f5ef3, ASSIGN 2024-11-14T12:54:33,941 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2201f29440b3aa3259741bbb374e209d, ASSIGN 2024-11-14T12:54:33,941 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure a572ae08c8738aa198c40f32aa4b5f96, server=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:33,941 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2201f29440b3aa3259741bbb374e209d, server=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:33,941 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=73, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:33,941 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 22b9f10374fc1ea7a876d476ce0b083d, server=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:33,941 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:33,942 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,942 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,942 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,942 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=669ef94e8b26,43475,1731588864980 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,942 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,942 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,942 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:33,942 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN 2024-11-14T12:54:33,942 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:33,942 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN 2024-11-14T12:54:33,942 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:33,942 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN 2024-11-14T12:54:33,942 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f16209fa220cf1b7a18b891e4fe53c6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:33,942 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f16209fa220cf1b7a18b891e4fe53c6, ASSIGN 2024-11-14T12:54:33,942 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22b9f10374fc1ea7a876d476ce0b083d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:33,942 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22b9f10374fc1ea7a876d476ce0b083d, ASSIGN 2024-11-14T12:54:33,942 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=18ddd925c95b1b23296976da04f97e83, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:33,943 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=18ddd925c95b1b23296976da04f97e83, ASSIGN 2024-11-14T12:54:33,943 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=333a3eba2bb2d318c041beb4642f5ef3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:33,943 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=333a3eba2bb2d318c041beb4642f5ef3, ASSIGN 2024-11-14T12:54:33,943 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2201f29440b3aa3259741bbb374e209d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:33,943 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2201f29440b3aa3259741bbb374e209d, ASSIGN 2024-11-14T12:54:33,943 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure a572ae08c8738aa198c40f32aa4b5f96, server=669ef94e8b26,43475,1731588864980 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,943 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2201f29440b3aa3259741bbb374e209d, server=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,943 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=73, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,943 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 22b9f10374fc1ea7a876d476ce0b083d, server=669ef94e8b26,43475,1731588864980 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,943 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 11 msec 2024-11-14T12:54:33,945 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=65 2024-11-14T12:54:33,945 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=669ef94e8b26,43475,1731588864980, table=hbase:meta, region=1588230740 2024-11-14T12:54:33,946 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 669ef94e8b26,43475,1731588864980, state=OPEN 2024-11-14T12:54:33,955 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45309-0x1013ab2e72b0000, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:33,955 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43475-0x1013ab2e72b0001, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:33,955 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46877-0x1013ab2e72b0002, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:33,955 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:33,955 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:33,955 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:33,957 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN to state=OFFLINE, location=null, table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7 to restore RIT 2024-11-14T12:54:33,957 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN to state=OFFLINE, location=null, table=test, region=a572ae08c8738aa198c40f32aa4b5f96 to restore RIT 2024-11-14T12:54:33,957 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN to state=OFFLINE, location=null, table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8 to restore RIT 2024-11-14T12:54:33,957 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f16209fa220cf1b7a18b891e4fe53c6, ASSIGN to state=OFFLINE, location=null, table=test, region=4f16209fa220cf1b7a18b891e4fe53c6 to restore RIT 2024-11-14T12:54:33,957 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22b9f10374fc1ea7a876d476ce0b083d, ASSIGN to state=OFFLINE, location=null, table=test, region=22b9f10374fc1ea7a876d476ce0b083d to restore RIT 2024-11-14T12:54:33,957 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=18ddd925c95b1b23296976da04f97e83, ASSIGN to state=OFFLINE, location=null, table=test, region=18ddd925c95b1b23296976da04f97e83 to restore RIT 2024-11-14T12:54:33,958 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=333a3eba2bb2d318c041beb4642f5ef3, ASSIGN to state=OFFLINE, location=null, table=test, region=333a3eba2bb2d318c041beb4642f5ef3 to restore RIT 2024-11-14T12:54:33,958 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2201f29440b3aa3259741bbb374e209d, ASSIGN to state=OFFLINE, location=null, table=test, region=2201f29440b3aa3259741bbb374e209d to restore RIT 2024-11-14T12:54:33,958 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731588903958 2024-11-14T12:54:33,958 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-14T12:54:33,958 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-14T12:54:33,960 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-14T12:54:33,964 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-14T12:54:33,964 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=77, ppid=73, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:33,964 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T12:54:33,964 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=77, ppid=73, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-14T12:54:33,967 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a572ae08c8738aa198c40f32aa4b5f96, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:33,967 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2ccfc8c17ab1571bc4ea6d526017f9f, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:33,968 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd0512a8a2f36befd3453e1140df3af, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:33,968 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b8044551b940cf51bfa7c7633c4df21d, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:33,968 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f7fbc112f1353900c1181e3ed36707, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=5 2024-11-14T12:54:33,968 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf47be17fd22bc204db28da07b30af2c, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:33,969 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f76ec25f51f69b1106342a7dfae65fb6, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:33,969 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cdc4b47ed9fe07b56101f9e0e17abcd7, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:33,969 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03f289a2805f0b2f037581e9390ba191, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:33,969 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1358fa84af33d1799bd11ba7f4959a37, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:33,970 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=333a3eba2bb2d318c041beb4642f5ef3, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:33,970 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64b5bd4e6112417a55e871b36551c5b8, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:33,970 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e90cc230c4f13fece446e3e8420dddbe, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:33,970 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f16209fa220cf1b7a18b891e4fe53c6, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:33,970 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=338b47e35e9a8b9dd3045d365aaf47bd, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:33,971 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ea6e95809b33f853859fa227179bfe85, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:33,971 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=18ddd925c95b1b23296976da04f97e83, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:33,971 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5d884ba0737bc572475e022a6ed1bc8, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:33,971 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d33b9e9c25fcfc654d257646677381c, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:33,971 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8ddc4f331afb3a3007ef5c49cdf6ab8, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:33,972 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8c0d62856832eec9f9ca20bb6db602c, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:33,972 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87eb074dbd46b3bc6e27fb0bf3291d5b, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:33,972 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22b9f10374fc1ea7a876d476ce0b083d, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:33,972 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cadcf21958eaec53ee489ca0918e3ab7, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:33,972 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66b34b9be40614bd68d0cea995e44193, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:33,973 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2201f29440b3aa3259741bbb374e209d, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:33,973 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-14T12:54:33,973 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731588933973 2024-11-14T12:54:33,973 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731588993973 2024-11-14T12:54:33,973 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 14 msec 2024-11-14T12:54:33,973 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-14T12:54:33,973 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T12:54:33,973 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T12:54:33,973 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T12:54:33,974 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-14T12:54:33,974 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-14T12:54:33,974 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-14T12:54:33,974 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T12:54:33,975 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-14T12:54:33,977 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-14T12:54:33,978 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T12:54:33,978 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T12:54:33,978 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:33,978 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:33,978 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T12:54:33,978 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:33,978 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:33,978 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:33,978 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:33,978 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:33,978 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T12:54:33,978 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:33,978 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T12:54:33,978 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T12:54:33,978 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T12:54:33,978 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:33,978 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T12:54:33,978 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T12:54:33,979 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-14T12:54:33,979 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=23f7fbc112f1353900c1181e3ed36707, REOPEN/MOVE 2024-11-14T12:54:33,982 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true 2024-11-14T12:54:33,982 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:33,982 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:33,982 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T12:54:33,982 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:33,982 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:33,983 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:33,983 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN 2024-11-14T12:54:33,983 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN 2024-11-14T12:54:33,983 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN 2024-11-14T12:54:33,983 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f16209fa220cf1b7a18b891e4fe53c6, ASSIGN 2024-11-14T12:54:33,983 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22b9f10374fc1ea7a876d476ce0b083d, ASSIGN 2024-11-14T12:54:33,983 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=18ddd925c95b1b23296976da04f97e83, ASSIGN 2024-11-14T12:54:33,983 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=333a3eba2bb2d318c041beb4642f5ef3, ASSIGN 2024-11-14T12:54:33,983 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2201f29440b3aa3259741bbb374e209d, ASSIGN 2024-11-14T12:54:33,983 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure a572ae08c8738aa198c40f32aa4b5f96, server=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:33,983 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2201f29440b3aa3259741bbb374e209d, server=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:33,983 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=73, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:33,983 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:33,984 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,984 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,984 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,984 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=669ef94e8b26,43475,1731588864980 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,984 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,984 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,984 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:33,984 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN 2024-11-14T12:54:33,984 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:33,984 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN 2024-11-14T12:54:33,984 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:33,984 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN 2024-11-14T12:54:33,984 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f16209fa220cf1b7a18b891e4fe53c6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:33,984 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f16209fa220cf1b7a18b891e4fe53c6, ASSIGN 2024-11-14T12:54:33,984 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22b9f10374fc1ea7a876d476ce0b083d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:33,984 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22b9f10374fc1ea7a876d476ce0b083d, ASSIGN 2024-11-14T12:54:33,984 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=18ddd925c95b1b23296976da04f97e83, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:33,985 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=18ddd925c95b1b23296976da04f97e83, ASSIGN 2024-11-14T12:54:33,985 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=333a3eba2bb2d318c041beb4642f5ef3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:33,985 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=333a3eba2bb2d318c041beb4642f5ef3, ASSIGN 2024-11-14T12:54:33,985 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2201f29440b3aa3259741bbb374e209d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:33,985 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2201f29440b3aa3259741bbb374e209d, ASSIGN 2024-11-14T12:54:33,985 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure a572ae08c8738aa198c40f32aa4b5f96, server=669ef94e8b26,43475,1731588864980 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,985 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2201f29440b3aa3259741bbb374e209d, server=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,985 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=73, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:33,985 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 10 msec 2024-11-14T12:54:33,986 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=65 2024-11-14T12:54:33,987 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=669ef94e8b26,43475,1731588864980, table=hbase:meta, region=1588230740 2024-11-14T12:54:33,987 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 669ef94e8b26,43475,1731588864980, state=OPEN 2024-11-14T12:54:33,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45309-0x1013ab2e72b0000, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:33,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43475-0x1013ab2e72b0001, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:33,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46877-0x1013ab2e72b0002, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:33,997 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:33,997 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:33,997 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:33,998 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN to state=OFFLINE, location=null, table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7 to restore RIT 2024-11-14T12:54:33,998 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN to state=OFFLINE, location=null, table=test, region=a572ae08c8738aa198c40f32aa4b5f96 to restore RIT 2024-11-14T12:54:33,998 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN to state=OFFLINE, location=null, table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8 to restore RIT 2024-11-14T12:54:33,998 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f16209fa220cf1b7a18b891e4fe53c6, ASSIGN to state=OFFLINE, location=null, table=test, region=4f16209fa220cf1b7a18b891e4fe53c6 to restore RIT 2024-11-14T12:54:33,999 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22b9f10374fc1ea7a876d476ce0b083d, ASSIGN to state=OFFLINE, location=null, table=test, region=22b9f10374fc1ea7a876d476ce0b083d to restore RIT 2024-11-14T12:54:33,999 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=18ddd925c95b1b23296976da04f97e83, ASSIGN to state=OFFLINE, location=null, table=test, region=18ddd925c95b1b23296976da04f97e83 to restore RIT 2024-11-14T12:54:33,999 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=333a3eba2bb2d318c041beb4642f5ef3, ASSIGN to state=OFFLINE, location=null, table=test, region=333a3eba2bb2d318c041beb4642f5ef3 to restore RIT 2024-11-14T12:54:33,999 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2201f29440b3aa3259741bbb374e209d, ASSIGN to state=OFFLINE, location=null, table=test, region=2201f29440b3aa3259741bbb374e209d to restore RIT 2024-11-14T12:54:33,999 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731588903999 2024-11-14T12:54:33,999 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-14T12:54:33,999 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-14T12:54:34,001 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-14T12:54:34,003 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-14T12:54:34,003 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=76, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2201f29440b3aa3259741bbb374e209d, server=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:34,003 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T12:54:34,003 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=76, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2201f29440b3aa3259741bbb374e209d, server=669ef94e8b26,46877,1731588865107 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-14T12:54:34,007 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a572ae08c8738aa198c40f32aa4b5f96, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,007 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2ccfc8c17ab1571bc4ea6d526017f9f, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,008 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd0512a8a2f36befd3453e1140df3af, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,008 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b8044551b940cf51bfa7c7633c4df21d, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,008 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f7fbc112f1353900c1181e3ed36707, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=5 2024-11-14T12:54:34,008 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf47be17fd22bc204db28da07b30af2c, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,008 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f76ec25f51f69b1106342a7dfae65fb6, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,008 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cdc4b47ed9fe07b56101f9e0e17abcd7, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,009 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03f289a2805f0b2f037581e9390ba191, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,009 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1358fa84af33d1799bd11ba7f4959a37, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,009 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=333a3eba2bb2d318c041beb4642f5ef3, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,009 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64b5bd4e6112417a55e871b36551c5b8, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,009 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e90cc230c4f13fece446e3e8420dddbe, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,009 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f16209fa220cf1b7a18b891e4fe53c6, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,010 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=338b47e35e9a8b9dd3045d365aaf47bd, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,010 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ea6e95809b33f853859fa227179bfe85, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,010 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=18ddd925c95b1b23296976da04f97e83, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,010 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5d884ba0737bc572475e022a6ed1bc8, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,010 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d33b9e9c25fcfc654d257646677381c, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,010 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8ddc4f331afb3a3007ef5c49cdf6ab8, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,011 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8c0d62856832eec9f9ca20bb6db602c, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,011 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87eb074dbd46b3bc6e27fb0bf3291d5b, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,011 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22b9f10374fc1ea7a876d476ce0b083d, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,011 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cadcf21958eaec53ee489ca0918e3ab7, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,011 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66b34b9be40614bd68d0cea995e44193, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,012 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2201f29440b3aa3259741bbb374e209d, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,012 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-14T12:54:34,012 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731588934012 2024-11-14T12:54:34,012 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731588994012 2024-11-14T12:54:34,012 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 12 msec 2024-11-14T12:54:34,012 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-14T12:54:34,012 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T12:54:34,012 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T12:54:34,012 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T12:54:34,013 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-14T12:54:34,013 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-14T12:54:34,013 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-14T12:54:34,013 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T12:54:34,014 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-14T12:54:34,018 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-14T12:54:34,019 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T12:54:34,019 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T12:54:34,019 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,019 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,019 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T12:54:34,019 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,019 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,019 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,019 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,019 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,019 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45309 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T12:54:34,019 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,020 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T12:54:34,020 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T12:54:34,020 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T12:54:34,020 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,020 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T12:54:34,020 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T12:54:34,021 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-14T12:54:34,021 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=23f7fbc112f1353900c1181e3ed36707, REOPEN/MOVE 2024-11-14T12:54:34,023 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true 2024-11-14T12:54:34,024 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:34,024 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:34,024 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T12:54:34,024 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:34,024 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:34,024 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:34,024 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN 2024-11-14T12:54:34,024 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN 2024-11-14T12:54:34,024 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN 2024-11-14T12:54:34,024 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f16209fa220cf1b7a18b891e4fe53c6, ASSIGN 2024-11-14T12:54:34,024 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22b9f10374fc1ea7a876d476ce0b083d, ASSIGN 2024-11-14T12:54:34,024 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=18ddd925c95b1b23296976da04f97e83, ASSIGN 2024-11-14T12:54:34,025 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=333a3eba2bb2d318c041beb4642f5ef3, ASSIGN 2024-11-14T12:54:34,025 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2201f29440b3aa3259741bbb374e209d, ASSIGN 2024-11-14T12:54:34,025 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure a572ae08c8738aa198c40f32aa4b5f96, server=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:34,025 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2201f29440b3aa3259741bbb374e209d, server=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:34,025 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,025 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,025 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,025 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,025 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=669ef94e8b26,43475,1731588864980 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,025 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,025 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,025 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,025 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN 2024-11-14T12:54:34,025 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,025 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN 2024-11-14T12:54:34,026 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,026 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN 2024-11-14T12:54:34,026 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f16209fa220cf1b7a18b891e4fe53c6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,026 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f16209fa220cf1b7a18b891e4fe53c6, ASSIGN 2024-11-14T12:54:34,026 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22b9f10374fc1ea7a876d476ce0b083d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,026 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22b9f10374fc1ea7a876d476ce0b083d, ASSIGN 2024-11-14T12:54:34,026 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=18ddd925c95b1b23296976da04f97e83, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,026 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=18ddd925c95b1b23296976da04f97e83, ASSIGN 2024-11-14T12:54:34,026 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=333a3eba2bb2d318c041beb4642f5ef3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,026 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=333a3eba2bb2d318c041beb4642f5ef3, ASSIGN 2024-11-14T12:54:34,026 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2201f29440b3aa3259741bbb374e209d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,026 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2201f29440b3aa3259741bbb374e209d, ASSIGN 2024-11-14T12:54:34,026 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure a572ae08c8738aa198c40f32aa4b5f96, server=669ef94e8b26,43475,1731588864980 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,026 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2201f29440b3aa3259741bbb374e209d, server=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,026 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 11 msec 2024-11-14T12:54:34,028 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=65 2024-11-14T12:54:34,028 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=669ef94e8b26,43475,1731588864980, table=hbase:meta, region=1588230740 2024-11-14T12:54:34,029 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 669ef94e8b26,43475,1731588864980, state=OPEN 2024-11-14T12:54:34,039 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45309-0x1013ab2e72b0000, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:34,039 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46877-0x1013ab2e72b0002, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:34,039 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43475-0x1013ab2e72b0001, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:34,039 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:34,039 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:34,039 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:34,039 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45309 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T12:54:34,040 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN to state=OFFLINE, location=null, table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7 to restore RIT 2024-11-14T12:54:34,040 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN to state=OFFLINE, location=null, table=test, region=a572ae08c8738aa198c40f32aa4b5f96 to restore RIT 2024-11-14T12:54:34,040 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN to state=OFFLINE, location=null, table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8 to restore RIT 2024-11-14T12:54:34,040 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f16209fa220cf1b7a18b891e4fe53c6, ASSIGN to state=OFFLINE, location=null, table=test, region=4f16209fa220cf1b7a18b891e4fe53c6 to restore RIT 2024-11-14T12:54:34,040 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22b9f10374fc1ea7a876d476ce0b083d, ASSIGN to state=OFFLINE, location=null, table=test, region=22b9f10374fc1ea7a876d476ce0b083d to restore RIT 2024-11-14T12:54:34,040 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=18ddd925c95b1b23296976da04f97e83, ASSIGN to state=OFFLINE, location=null, table=test, region=18ddd925c95b1b23296976da04f97e83 to restore RIT 2024-11-14T12:54:34,041 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=333a3eba2bb2d318c041beb4642f5ef3, ASSIGN to state=OFFLINE, location=null, table=test, region=333a3eba2bb2d318c041beb4642f5ef3 to restore RIT 2024-11-14T12:54:34,041 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2201f29440b3aa3259741bbb374e209d, ASSIGN to state=OFFLINE, location=null, table=test, region=2201f29440b3aa3259741bbb374e209d to restore RIT 2024-11-14T12:54:34,041 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731588904041 2024-11-14T12:54:34,041 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-14T12:54:34,041 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-14T12:54:34,043 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-14T12:54:34,044 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-14T12:54:34,044 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure a572ae08c8738aa198c40f32aa4b5f96, server=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:34,044 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T12:54:34,044 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure a572ae08c8738aa198c40f32aa4b5f96, server=669ef94e8b26,43475,1731588864980 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-14T12:54:34,048 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a572ae08c8738aa198c40f32aa4b5f96, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,048 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2ccfc8c17ab1571bc4ea6d526017f9f, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,048 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd0512a8a2f36befd3453e1140df3af, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,049 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b8044551b940cf51bfa7c7633c4df21d, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,049 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f7fbc112f1353900c1181e3ed36707, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=5 2024-11-14T12:54:34,049 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf47be17fd22bc204db28da07b30af2c, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,049 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f76ec25f51f69b1106342a7dfae65fb6, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,049 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cdc4b47ed9fe07b56101f9e0e17abcd7, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,049 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03f289a2805f0b2f037581e9390ba191, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,049 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1358fa84af33d1799bd11ba7f4959a37, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,049 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=333a3eba2bb2d318c041beb4642f5ef3, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,050 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64b5bd4e6112417a55e871b36551c5b8, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,050 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e90cc230c4f13fece446e3e8420dddbe, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,050 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f16209fa220cf1b7a18b891e4fe53c6, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,050 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=338b47e35e9a8b9dd3045d365aaf47bd, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,050 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ea6e95809b33f853859fa227179bfe85, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,050 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=18ddd925c95b1b23296976da04f97e83, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,050 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5d884ba0737bc572475e022a6ed1bc8, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,051 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d33b9e9c25fcfc654d257646677381c, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,051 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8ddc4f331afb3a3007ef5c49cdf6ab8, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,051 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8c0d62856832eec9f9ca20bb6db602c, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,051 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87eb074dbd46b3bc6e27fb0bf3291d5b, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,051 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22b9f10374fc1ea7a876d476ce0b083d, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,052 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cadcf21958eaec53ee489ca0918e3ab7, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,052 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66b34b9be40614bd68d0cea995e44193, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,052 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2201f29440b3aa3259741bbb374e209d, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,052 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-14T12:54:34,052 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731588934052 2024-11-14T12:54:34,052 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731588994052 2024-11-14T12:54:34,052 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 11 msec 2024-11-14T12:54:34,052 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-14T12:54:34,052 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T12:54:34,052 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T12:54:34,053 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T12:54:34,053 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-14T12:54:34,053 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-14T12:54:34,053 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-14T12:54:34,053 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T12:54:34,054 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-14T12:54:34,056 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-14T12:54:34,056 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T12:54:34,056 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T12:54:34,056 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,056 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,056 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T12:54:34,056 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,057 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,057 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,057 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,057 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,057 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,057 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T12:54:34,057 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T12:54:34,057 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,057 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T12:54:34,057 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T12:54:34,058 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-14T12:54:34,058 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=23f7fbc112f1353900c1181e3ed36707, REOPEN/MOVE 2024-11-14T12:54:34,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true 2024-11-14T12:54:34,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:34,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:34,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T12:54:34,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:34,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:34,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:34,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN 2024-11-14T12:54:34,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN 2024-11-14T12:54:34,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN 2024-11-14T12:54:34,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f16209fa220cf1b7a18b891e4fe53c6, ASSIGN 2024-11-14T12:54:34,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22b9f10374fc1ea7a876d476ce0b083d, ASSIGN 2024-11-14T12:54:34,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=18ddd925c95b1b23296976da04f97e83, ASSIGN 2024-11-14T12:54:34,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=333a3eba2bb2d318c041beb4642f5ef3, ASSIGN 2024-11-14T12:54:34,062 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2201f29440b3aa3259741bbb374e209d, ASSIGN 2024-11-14T12:54:34,062 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure a572ae08c8738aa198c40f32aa4b5f96, server=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:34,062 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,062 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,062 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,062 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,062 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=669ef94e8b26,43475,1731588864980 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,062 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,062 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,062 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,062 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN 2024-11-14T12:54:34,062 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,062 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN 2024-11-14T12:54:34,062 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,062 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN 2024-11-14T12:54:34,062 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f16209fa220cf1b7a18b891e4fe53c6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,062 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f16209fa220cf1b7a18b891e4fe53c6, ASSIGN 2024-11-14T12:54:34,062 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22b9f10374fc1ea7a876d476ce0b083d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,062 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22b9f10374fc1ea7a876d476ce0b083d, ASSIGN 2024-11-14T12:54:34,062 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=18ddd925c95b1b23296976da04f97e83, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,062 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=18ddd925c95b1b23296976da04f97e83, ASSIGN 2024-11-14T12:54:34,063 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=333a3eba2bb2d318c041beb4642f5ef3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,063 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=333a3eba2bb2d318c041beb4642f5ef3, ASSIGN 2024-11-14T12:54:34,063 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2201f29440b3aa3259741bbb374e209d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,063 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2201f29440b3aa3259741bbb374e209d, ASSIGN 2024-11-14T12:54:34,063 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure a572ae08c8738aa198c40f32aa4b5f96, server=669ef94e8b26,43475,1731588864980 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,063 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-11-14T12:54:34,064 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=65 2024-11-14T12:54:34,064 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=669ef94e8b26,43475,1731588864980, table=hbase:meta, region=1588230740 2024-11-14T12:54:34,064 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 669ef94e8b26,43475,1731588864980, state=OPEN 2024-11-14T12:54:34,072 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46877-0x1013ab2e72b0002, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:34,072 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43475-0x1013ab2e72b0001, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:34,072 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45309-0x1013ab2e72b0000, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:34,072 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:34,072 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:34,072 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:34,072 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN to state=OFFLINE, location=null, table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7 to restore RIT 2024-11-14T12:54:34,073 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN to state=OFFLINE, location=null, table=test, region=a572ae08c8738aa198c40f32aa4b5f96 to restore RIT 2024-11-14T12:54:34,073 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN to state=OFFLINE, location=null, table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8 to restore RIT 2024-11-14T12:54:34,073 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f16209fa220cf1b7a18b891e4fe53c6, ASSIGN to state=OFFLINE, location=null, table=test, region=4f16209fa220cf1b7a18b891e4fe53c6 to restore RIT 2024-11-14T12:54:34,073 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22b9f10374fc1ea7a876d476ce0b083d, ASSIGN to state=OFFLINE, location=null, table=test, region=22b9f10374fc1ea7a876d476ce0b083d to restore RIT 2024-11-14T12:54:34,073 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=18ddd925c95b1b23296976da04f97e83, ASSIGN to state=OFFLINE, location=null, table=test, region=18ddd925c95b1b23296976da04f97e83 to restore RIT 2024-11-14T12:54:34,073 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=333a3eba2bb2d318c041beb4642f5ef3, ASSIGN to state=OFFLINE, location=null, table=test, region=333a3eba2bb2d318c041beb4642f5ef3 to restore RIT 2024-11-14T12:54:34,073 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2201f29440b3aa3259741bbb374e209d, ASSIGN to state=OFFLINE, location=null, table=test, region=2201f29440b3aa3259741bbb374e209d to restore RIT 2024-11-14T12:54:34,074 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731588904074 2024-11-14T12:54:34,074 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-14T12:54:34,074 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-14T12:54:34,075 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-14T12:54:34,077 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-14T12:54:34,077 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2201f29440b3aa3259741bbb374e209d, ASSIGN 2024-11-14T12:54:34,077 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T12:54:34,077 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2201f29440b3aa3259741bbb374e209d, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-14T12:54:34,081 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a572ae08c8738aa198c40f32aa4b5f96, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,081 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2ccfc8c17ab1571bc4ea6d526017f9f, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,081 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd0512a8a2f36befd3453e1140df3af, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,081 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b8044551b940cf51bfa7c7633c4df21d, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,082 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f7fbc112f1353900c1181e3ed36707, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=5 2024-11-14T12:54:34,082 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf47be17fd22bc204db28da07b30af2c, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,082 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f76ec25f51f69b1106342a7dfae65fb6, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,082 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cdc4b47ed9fe07b56101f9e0e17abcd7, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,082 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03f289a2805f0b2f037581e9390ba191, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,082 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1358fa84af33d1799bd11ba7f4959a37, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,082 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=333a3eba2bb2d318c041beb4642f5ef3, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64b5bd4e6112417a55e871b36551c5b8, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e90cc230c4f13fece446e3e8420dddbe, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f16209fa220cf1b7a18b891e4fe53c6, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=338b47e35e9a8b9dd3045d365aaf47bd, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ea6e95809b33f853859fa227179bfe85, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=18ddd925c95b1b23296976da04f97e83, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5d884ba0737bc572475e022a6ed1bc8, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,084 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d33b9e9c25fcfc654d257646677381c, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,084 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8ddc4f331afb3a3007ef5c49cdf6ab8, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,084 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8c0d62856832eec9f9ca20bb6db602c, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,084 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87eb074dbd46b3bc6e27fb0bf3291d5b, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,084 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22b9f10374fc1ea7a876d476ce0b083d, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,085 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cadcf21958eaec53ee489ca0918e3ab7, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,085 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66b34b9be40614bd68d0cea995e44193, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,085 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2201f29440b3aa3259741bbb374e209d, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,085 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-14T12:54:34,085 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731588934085 2024-11-14T12:54:34,085 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731588994085 2024-11-14T12:54:34,085 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 11 msec 2024-11-14T12:54:34,085 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-14T12:54:34,085 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T12:54:34,086 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T12:54:34,086 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T12:54:34,086 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-14T12:54:34,086 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-14T12:54:34,086 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-14T12:54:34,086 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T12:54:34,087 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-14T12:54:34,089 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-14T12:54:34,089 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T12:54:34,089 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T12:54:34,089 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,089 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,089 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,090 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,090 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,090 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,090 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,090 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,090 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T12:54:34,090 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T12:54:34,090 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,090 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T12:54:34,090 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T12:54:34,091 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-14T12:54:34,091 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=23f7fbc112f1353900c1181e3ed36707, REOPEN/MOVE 2024-11-14T12:54:34,093 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true 2024-11-14T12:54:34,093 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:34,093 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:34,094 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T12:54:34,094 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:34,094 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:34,094 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:34,094 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN 2024-11-14T12:54:34,094 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN 2024-11-14T12:54:34,094 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN 2024-11-14T12:54:34,094 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f16209fa220cf1b7a18b891e4fe53c6, ASSIGN 2024-11-14T12:54:34,094 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22b9f10374fc1ea7a876d476ce0b083d, ASSIGN 2024-11-14T12:54:34,094 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=18ddd925c95b1b23296976da04f97e83, ASSIGN 2024-11-14T12:54:34,094 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=333a3eba2bb2d318c041beb4642f5ef3, ASSIGN 2024-11-14T12:54:34,094 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2201f29440b3aa3259741bbb374e209d, ASSIGN 2024-11-14T12:54:34,094 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,094 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,094 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,094 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,094 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=669ef94e8b26,43475,1731588864980 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,094 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,094 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,094 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,094 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN 2024-11-14T12:54:34,094 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,094 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN 2024-11-14T12:54:34,094 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,095 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN 2024-11-14T12:54:34,095 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f16209fa220cf1b7a18b891e4fe53c6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,095 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f16209fa220cf1b7a18b891e4fe53c6, ASSIGN 2024-11-14T12:54:34,095 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22b9f10374fc1ea7a876d476ce0b083d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,095 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22b9f10374fc1ea7a876d476ce0b083d, ASSIGN 2024-11-14T12:54:34,095 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=18ddd925c95b1b23296976da04f97e83, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,095 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=18ddd925c95b1b23296976da04f97e83, ASSIGN 2024-11-14T12:54:34,095 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=333a3eba2bb2d318c041beb4642f5ef3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,095 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=333a3eba2bb2d318c041beb4642f5ef3, ASSIGN 2024-11-14T12:54:34,095 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2201f29440b3aa3259741bbb374e209d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,095 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2201f29440b3aa3259741bbb374e209d, ASSIGN 2024-11-14T12:54:34,095 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-14T12:54:34,099 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=65 2024-11-14T12:54:34,100 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=669ef94e8b26,43475,1731588864980, table=hbase:meta, region=1588230740 2024-11-14T12:54:34,100 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 669ef94e8b26,43475,1731588864980, state=OPEN 2024-11-14T12:54:34,105 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43475-0x1013ab2e72b0001, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:34,105 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45309-0x1013ab2e72b0000, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:34,105 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46877-0x1013ab2e72b0002, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:34,105 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:34,105 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:34,105 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:34,106 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN to state=OFFLINE, location=null, table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7 to restore RIT 2024-11-14T12:54:34,106 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN to state=OFFLINE, location=null, table=test, region=a572ae08c8738aa198c40f32aa4b5f96 to restore RIT 2024-11-14T12:54:34,106 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN to state=OFFLINE, location=null, table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8 to restore RIT 2024-11-14T12:54:34,106 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f16209fa220cf1b7a18b891e4fe53c6, ASSIGN to state=OFFLINE, location=null, table=test, region=4f16209fa220cf1b7a18b891e4fe53c6 to restore RIT 2024-11-14T12:54:34,106 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22b9f10374fc1ea7a876d476ce0b083d, ASSIGN to state=OFFLINE, location=null, table=test, region=22b9f10374fc1ea7a876d476ce0b083d to restore RIT 2024-11-14T12:54:34,106 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=18ddd925c95b1b23296976da04f97e83, ASSIGN to state=OFFLINE, location=null, table=test, region=18ddd925c95b1b23296976da04f97e83 to restore RIT 2024-11-14T12:54:34,106 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=333a3eba2bb2d318c041beb4642f5ef3, ASSIGN to state=OFFLINE, location=null, table=test, region=333a3eba2bb2d318c041beb4642f5ef3 to restore RIT 2024-11-14T12:54:34,106 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2201f29440b3aa3259741bbb374e209d, ASSIGN to state=OFFLINE, location=null, table=test, region=2201f29440b3aa3259741bbb374e209d to restore RIT 2024-11-14T12:54:34,111 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731588904111 2024-11-14T12:54:34,111 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-14T12:54:34,111 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-14T12:54:34,113 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-14T12:54:34,116 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-14T12:54:34,116 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=333a3eba2bb2d318c041beb4642f5ef3, ASSIGN 2024-11-14T12:54:34,116 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T12:54:34,116 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=333a3eba2bb2d318c041beb4642f5ef3, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-14T12:54:34,118 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a572ae08c8738aa198c40f32aa4b5f96, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,118 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2ccfc8c17ab1571bc4ea6d526017f9f, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,118 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd0512a8a2f36befd3453e1140df3af, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,118 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b8044551b940cf51bfa7c7633c4df21d, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,118 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f7fbc112f1353900c1181e3ed36707, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=5 2024-11-14T12:54:34,118 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf47be17fd22bc204db28da07b30af2c, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,119 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f76ec25f51f69b1106342a7dfae65fb6, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,119 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cdc4b47ed9fe07b56101f9e0e17abcd7, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,119 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03f289a2805f0b2f037581e9390ba191, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,119 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1358fa84af33d1799bd11ba7f4959a37, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,119 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=333a3eba2bb2d318c041beb4642f5ef3, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,119 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64b5bd4e6112417a55e871b36551c5b8, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,119 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e90cc230c4f13fece446e3e8420dddbe, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,119 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f16209fa220cf1b7a18b891e4fe53c6, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,120 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=338b47e35e9a8b9dd3045d365aaf47bd, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,120 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ea6e95809b33f853859fa227179bfe85, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,120 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=18ddd925c95b1b23296976da04f97e83, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,120 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5d884ba0737bc572475e022a6ed1bc8, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,120 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d33b9e9c25fcfc654d257646677381c, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,120 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8ddc4f331afb3a3007ef5c49cdf6ab8, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,120 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8c0d62856832eec9f9ca20bb6db602c, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,121 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87eb074dbd46b3bc6e27fb0bf3291d5b, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,121 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22b9f10374fc1ea7a876d476ce0b083d, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,121 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cadcf21958eaec53ee489ca0918e3ab7, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,121 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66b34b9be40614bd68d0cea995e44193, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,121 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2201f29440b3aa3259741bbb374e209d, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,121 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-14T12:54:34,121 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731588934121 2024-11-14T12:54:34,122 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731588994122 2024-11-14T12:54:34,122 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 10 msec 2024-11-14T12:54:34,122 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-14T12:54:34,122 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T12:54:34,122 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T12:54:34,122 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T12:54:34,123 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-14T12:54:34,123 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-14T12:54:34,123 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-14T12:54:34,124 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T12:54:34,124 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45309 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T12:54:34,124 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-14T12:54:34,127 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-14T12:54:34,127 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T12:54:34,127 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T12:54:34,127 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,127 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,127 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,127 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,127 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,127 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,127 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,127 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T12:54:34,127 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T12:54:34,127 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,127 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T12:54:34,128 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T12:54:34,128 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-14T12:54:34,129 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=23f7fbc112f1353900c1181e3ed36707, REOPEN/MOVE 2024-11-14T12:54:34,130 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true 2024-11-14T12:54:34,130 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:34,130 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:34,130 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T12:54:34,131 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:34,131 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:34,131 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:34,131 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN 2024-11-14T12:54:34,131 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN 2024-11-14T12:54:34,131 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN 2024-11-14T12:54:34,131 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f16209fa220cf1b7a18b891e4fe53c6, ASSIGN 2024-11-14T12:54:34,131 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22b9f10374fc1ea7a876d476ce0b083d, ASSIGN 2024-11-14T12:54:34,131 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=18ddd925c95b1b23296976da04f97e83, ASSIGN 2024-11-14T12:54:34,131 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=333a3eba2bb2d318c041beb4642f5ef3, ASSIGN 2024-11-14T12:54:34,131 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,131 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,131 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,131 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,131 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=669ef94e8b26,43475,1731588864980 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,131 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,132 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,132 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,132 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN 2024-11-14T12:54:34,132 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,132 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN 2024-11-14T12:54:34,132 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,132 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN 2024-11-14T12:54:34,132 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f16209fa220cf1b7a18b891e4fe53c6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,132 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f16209fa220cf1b7a18b891e4fe53c6, ASSIGN 2024-11-14T12:54:34,132 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22b9f10374fc1ea7a876d476ce0b083d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,132 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22b9f10374fc1ea7a876d476ce0b083d, ASSIGN 2024-11-14T12:54:34,132 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=18ddd925c95b1b23296976da04f97e83, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,132 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=18ddd925c95b1b23296976da04f97e83, ASSIGN 2024-11-14T12:54:34,132 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=333a3eba2bb2d318c041beb4642f5ef3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,132 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=333a3eba2bb2d318c041beb4642f5ef3, ASSIGN 2024-11-14T12:54:34,132 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-14T12:54:34,133 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=65 2024-11-14T12:54:34,133 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=669ef94e8b26,43475,1731588864980, table=hbase:meta, region=1588230740 2024-11-14T12:54:34,133 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 669ef94e8b26,43475,1731588864980, state=OPEN 2024-11-14T12:54:34,138 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43475-0x1013ab2e72b0001, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:34,138 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46877-0x1013ab2e72b0002, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:34,138 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45309-0x1013ab2e72b0000, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:34,138 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:34,138 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:34,139 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:34,140 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN to state=OFFLINE, location=null, table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7 to restore RIT 2024-11-14T12:54:34,140 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN to state=OFFLINE, location=null, table=test, region=a572ae08c8738aa198c40f32aa4b5f96 to restore RIT 2024-11-14T12:54:34,140 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN to state=OFFLINE, location=null, table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8 to restore RIT 2024-11-14T12:54:34,140 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f16209fa220cf1b7a18b891e4fe53c6, ASSIGN to state=OFFLINE, location=null, table=test, region=4f16209fa220cf1b7a18b891e4fe53c6 to restore RIT 2024-11-14T12:54:34,140 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22b9f10374fc1ea7a876d476ce0b083d, ASSIGN to state=OFFLINE, location=null, table=test, region=22b9f10374fc1ea7a876d476ce0b083d to restore RIT 2024-11-14T12:54:34,140 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=18ddd925c95b1b23296976da04f97e83, ASSIGN to state=OFFLINE, location=null, table=test, region=18ddd925c95b1b23296976da04f97e83 to restore RIT 2024-11-14T12:54:34,140 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=333a3eba2bb2d318c041beb4642f5ef3, ASSIGN to state=OFFLINE, location=null, table=test, region=333a3eba2bb2d318c041beb4642f5ef3 to restore RIT 2024-11-14T12:54:34,141 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731588904140 2024-11-14T12:54:34,141 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-14T12:54:34,141 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-14T12:54:34,143 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-14T12:54:34,144 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-14T12:54:34,144 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=18ddd925c95b1b23296976da04f97e83, ASSIGN 2024-11-14T12:54:34,144 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T12:54:34,144 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=18ddd925c95b1b23296976da04f97e83, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-14T12:54:34,148 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a572ae08c8738aa198c40f32aa4b5f96, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,148 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2ccfc8c17ab1571bc4ea6d526017f9f, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,148 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd0512a8a2f36befd3453e1140df3af, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,148 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b8044551b940cf51bfa7c7633c4df21d, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,148 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f7fbc112f1353900c1181e3ed36707, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=5 2024-11-14T12:54:34,148 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf47be17fd22bc204db28da07b30af2c, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,149 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f76ec25f51f69b1106342a7dfae65fb6, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,149 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cdc4b47ed9fe07b56101f9e0e17abcd7, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,149 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03f289a2805f0b2f037581e9390ba191, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,149 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1358fa84af33d1799bd11ba7f4959a37, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,149 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=333a3eba2bb2d318c041beb4642f5ef3, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,149 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64b5bd4e6112417a55e871b36551c5b8, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e90cc230c4f13fece446e3e8420dddbe, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f16209fa220cf1b7a18b891e4fe53c6, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=338b47e35e9a8b9dd3045d365aaf47bd, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ea6e95809b33f853859fa227179bfe85, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=18ddd925c95b1b23296976da04f97e83, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5d884ba0737bc572475e022a6ed1bc8, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,151 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d33b9e9c25fcfc654d257646677381c, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,151 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8ddc4f331afb3a3007ef5c49cdf6ab8, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,151 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8c0d62856832eec9f9ca20bb6db602c, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,151 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87eb074dbd46b3bc6e27fb0bf3291d5b, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,151 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22b9f10374fc1ea7a876d476ce0b083d, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,151 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cadcf21958eaec53ee489ca0918e3ab7, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,152 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66b34b9be40614bd68d0cea995e44193, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,152 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2201f29440b3aa3259741bbb374e209d, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,152 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-14T12:54:34,152 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731588934152 2024-11-14T12:54:34,152 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731588994152 2024-11-14T12:54:34,152 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 11 msec 2024-11-14T12:54:34,152 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-14T12:54:34,152 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T12:54:34,152 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T12:54:34,153 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T12:54:34,153 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-14T12:54:34,153 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-14T12:54:34,153 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-14T12:54:34,153 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T12:54:34,154 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-14T12:54:34,156 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-14T12:54:34,156 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T12:54:34,156 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T12:54:34,156 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,156 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,156 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,156 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,157 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,157 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,157 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T12:54:34,157 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T12:54:34,157 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,157 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T12:54:34,157 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T12:54:34,158 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-14T12:54:34,158 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=23f7fbc112f1353900c1181e3ed36707, REOPEN/MOVE 2024-11-14T12:54:34,159 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true 2024-11-14T12:54:34,159 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:34,160 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:34,160 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T12:54:34,160 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:34,160 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:34,160 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:34,160 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN 2024-11-14T12:54:34,160 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN 2024-11-14T12:54:34,160 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN 2024-11-14T12:54:34,160 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f16209fa220cf1b7a18b891e4fe53c6, ASSIGN 2024-11-14T12:54:34,160 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22b9f10374fc1ea7a876d476ce0b083d, ASSIGN 2024-11-14T12:54:34,160 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=18ddd925c95b1b23296976da04f97e83, ASSIGN 2024-11-14T12:54:34,160 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,160 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,160 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,160 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,160 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=669ef94e8b26,43475,1731588864980 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,160 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,160 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,160 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,160 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN 2024-11-14T12:54:34,160 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,160 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN 2024-11-14T12:54:34,161 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,161 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN 2024-11-14T12:54:34,161 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f16209fa220cf1b7a18b891e4fe53c6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,161 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f16209fa220cf1b7a18b891e4fe53c6, ASSIGN 2024-11-14T12:54:34,161 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22b9f10374fc1ea7a876d476ce0b083d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,161 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22b9f10374fc1ea7a876d476ce0b083d, ASSIGN 2024-11-14T12:54:34,161 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=18ddd925c95b1b23296976da04f97e83, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,161 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=18ddd925c95b1b23296976da04f97e83, ASSIGN 2024-11-14T12:54:34,161 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-14T12:54:34,162 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=65 2024-11-14T12:54:34,162 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=669ef94e8b26,43475,1731588864980, table=hbase:meta, region=1588230740 2024-11-14T12:54:34,162 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 669ef94e8b26,43475,1731588864980, state=OPEN 2024-11-14T12:54:34,172 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43475-0x1013ab2e72b0001, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:34,172 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46877-0x1013ab2e72b0002, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:34,172 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45309-0x1013ab2e72b0000, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:34,172 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:34,172 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:34,172 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:34,174 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN to state=OFFLINE, location=null, table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7 to restore RIT 2024-11-14T12:54:34,174 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN to state=OFFLINE, location=null, table=test, region=a572ae08c8738aa198c40f32aa4b5f96 to restore RIT 2024-11-14T12:54:34,174 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN to state=OFFLINE, location=null, table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8 to restore RIT 2024-11-14T12:54:34,174 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f16209fa220cf1b7a18b891e4fe53c6, ASSIGN to state=OFFLINE, location=null, table=test, region=4f16209fa220cf1b7a18b891e4fe53c6 to restore RIT 2024-11-14T12:54:34,174 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22b9f10374fc1ea7a876d476ce0b083d, ASSIGN to state=OFFLINE, location=null, table=test, region=22b9f10374fc1ea7a876d476ce0b083d to restore RIT 2024-11-14T12:54:34,174 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=18ddd925c95b1b23296976da04f97e83, ASSIGN to state=OFFLINE, location=null, table=test, region=18ddd925c95b1b23296976da04f97e83 to restore RIT 2024-11-14T12:54:34,175 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731588904175 2024-11-14T12:54:34,175 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-14T12:54:34,175 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-14T12:54:34,177 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-14T12:54:34,178 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-14T12:54:34,178 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22b9f10374fc1ea7a876d476ce0b083d, ASSIGN 2024-11-14T12:54:34,178 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T12:54:34,178 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22b9f10374fc1ea7a876d476ce0b083d, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-14T12:54:34,181 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a572ae08c8738aa198c40f32aa4b5f96, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,182 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2ccfc8c17ab1571bc4ea6d526017f9f, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,182 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd0512a8a2f36befd3453e1140df3af, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,182 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b8044551b940cf51bfa7c7633c4df21d, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,182 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f7fbc112f1353900c1181e3ed36707, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=5 2024-11-14T12:54:34,182 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf47be17fd22bc204db28da07b30af2c, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,182 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f76ec25f51f69b1106342a7dfae65fb6, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,182 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cdc4b47ed9fe07b56101f9e0e17abcd7, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,183 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03f289a2805f0b2f037581e9390ba191, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,183 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1358fa84af33d1799bd11ba7f4959a37, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,183 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=333a3eba2bb2d318c041beb4642f5ef3, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,183 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64b5bd4e6112417a55e871b36551c5b8, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,183 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e90cc230c4f13fece446e3e8420dddbe, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,183 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f16209fa220cf1b7a18b891e4fe53c6, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,183 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=338b47e35e9a8b9dd3045d365aaf47bd, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,183 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ea6e95809b33f853859fa227179bfe85, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,184 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=18ddd925c95b1b23296976da04f97e83, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,184 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5d884ba0737bc572475e022a6ed1bc8, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,184 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d33b9e9c25fcfc654d257646677381c, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,184 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8ddc4f331afb3a3007ef5c49cdf6ab8, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,184 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8c0d62856832eec9f9ca20bb6db602c, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,184 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87eb074dbd46b3bc6e27fb0bf3291d5b, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,184 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22b9f10374fc1ea7a876d476ce0b083d, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,185 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cadcf21958eaec53ee489ca0918e3ab7, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,185 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66b34b9be40614bd68d0cea995e44193, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,185 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2201f29440b3aa3259741bbb374e209d, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,185 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-14T12:54:34,185 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731588934185 2024-11-14T12:54:34,185 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731588994185 2024-11-14T12:54:34,185 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-14T12:54:34,185 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-14T12:54:34,185 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T12:54:34,185 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T12:54:34,185 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T12:54:34,186 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-14T12:54:34,186 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-14T12:54:34,186 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-14T12:54:34,186 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T12:54:34,187 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-14T12:54:34,189 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-14T12:54:34,189 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T12:54:34,189 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T12:54:34,189 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,189 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,189 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,190 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,190 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,190 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T12:54:34,190 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T12:54:34,190 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,190 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T12:54:34,190 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T12:54:34,191 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-14T12:54:34,191 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=23f7fbc112f1353900c1181e3ed36707, REOPEN/MOVE 2024-11-14T12:54:34,192 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true 2024-11-14T12:54:34,192 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:34,192 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:34,192 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T12:54:34,192 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:34,192 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:34,192 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:34,192 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN 2024-11-14T12:54:34,192 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN 2024-11-14T12:54:34,192 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN 2024-11-14T12:54:34,193 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f16209fa220cf1b7a18b891e4fe53c6, ASSIGN 2024-11-14T12:54:34,193 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22b9f10374fc1ea7a876d476ce0b083d, ASSIGN 2024-11-14T12:54:34,193 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,193 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,193 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,193 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,193 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=669ef94e8b26,43475,1731588864980 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,193 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,193 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,193 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,193 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN 2024-11-14T12:54:34,193 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,193 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN 2024-11-14T12:54:34,193 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,193 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN 2024-11-14T12:54:34,193 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f16209fa220cf1b7a18b891e4fe53c6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,193 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f16209fa220cf1b7a18b891e4fe53c6, ASSIGN 2024-11-14T12:54:34,194 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22b9f10374fc1ea7a876d476ce0b083d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,194 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22b9f10374fc1ea7a876d476ce0b083d, ASSIGN 2024-11-14T12:54:34,194 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-14T12:54:34,195 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=65 2024-11-14T12:54:34,195 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=669ef94e8b26,43475,1731588864980, table=hbase:meta, region=1588230740 2024-11-14T12:54:34,195 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 669ef94e8b26,43475,1731588864980, state=OPEN 2024-11-14T12:54:34,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45309-0x1013ab2e72b0000, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:34,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43475-0x1013ab2e72b0001, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:34,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46877-0x1013ab2e72b0002, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:34,205 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:34,205 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:34,205 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:34,206 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN to state=OFFLINE, location=null, table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7 to restore RIT 2024-11-14T12:54:34,206 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN to state=OFFLINE, location=null, table=test, region=a572ae08c8738aa198c40f32aa4b5f96 to restore RIT 2024-11-14T12:54:34,206 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN to state=OFFLINE, location=null, table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8 to restore RIT 2024-11-14T12:54:34,207 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f16209fa220cf1b7a18b891e4fe53c6, ASSIGN to state=OFFLINE, location=null, table=test, region=4f16209fa220cf1b7a18b891e4fe53c6 to restore RIT 2024-11-14T12:54:34,207 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=22b9f10374fc1ea7a876d476ce0b083d, ASSIGN to state=OFFLINE, location=null, table=test, region=22b9f10374fc1ea7a876d476ce0b083d to restore RIT 2024-11-14T12:54:34,207 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731588904207 2024-11-14T12:54:34,207 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-14T12:54:34,207 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-14T12:54:34,209 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-14T12:54:34,210 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-14T12:54:34,210 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f16209fa220cf1b7a18b891e4fe53c6, ASSIGN 2024-11-14T12:54:34,211 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T12:54:34,211 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f16209fa220cf1b7a18b891e4fe53c6, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-14T12:54:34,214 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a572ae08c8738aa198c40f32aa4b5f96, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,214 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2ccfc8c17ab1571bc4ea6d526017f9f, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,214 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd0512a8a2f36befd3453e1140df3af, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,214 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b8044551b940cf51bfa7c7633c4df21d, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,214 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f7fbc112f1353900c1181e3ed36707, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=5 2024-11-14T12:54:34,215 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf47be17fd22bc204db28da07b30af2c, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,215 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f76ec25f51f69b1106342a7dfae65fb6, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,215 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cdc4b47ed9fe07b56101f9e0e17abcd7, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,215 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03f289a2805f0b2f037581e9390ba191, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,215 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1358fa84af33d1799bd11ba7f4959a37, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,215 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=333a3eba2bb2d318c041beb4642f5ef3, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,216 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64b5bd4e6112417a55e871b36551c5b8, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,216 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e90cc230c4f13fece446e3e8420dddbe, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,216 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f16209fa220cf1b7a18b891e4fe53c6, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,216 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=338b47e35e9a8b9dd3045d365aaf47bd, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,216 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ea6e95809b33f853859fa227179bfe85, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,216 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=18ddd925c95b1b23296976da04f97e83, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,217 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5d884ba0737bc572475e022a6ed1bc8, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,217 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d33b9e9c25fcfc654d257646677381c, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,217 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8ddc4f331afb3a3007ef5c49cdf6ab8, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,217 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8c0d62856832eec9f9ca20bb6db602c, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,217 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87eb074dbd46b3bc6e27fb0bf3291d5b, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,217 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22b9f10374fc1ea7a876d476ce0b083d, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,217 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cadcf21958eaec53ee489ca0918e3ab7, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,217 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66b34b9be40614bd68d0cea995e44193, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,218 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2201f29440b3aa3259741bbb374e209d, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,218 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-14T12:54:34,218 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731588934218 2024-11-14T12:54:34,218 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731588994218 2024-11-14T12:54:34,218 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 10 msec 2024-11-14T12:54:34,218 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-14T12:54:34,218 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T12:54:34,218 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T12:54:34,218 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T12:54:34,219 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-14T12:54:34,219 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-14T12:54:34,219 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-14T12:54:34,219 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T12:54:34,220 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-14T12:54:34,223 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-14T12:54:34,223 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T12:54:34,223 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T12:54:34,223 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,223 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,223 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,223 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,223 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T12:54:34,223 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T12:54:34,223 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,223 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T12:54:34,223 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T12:54:34,224 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-14T12:54:34,224 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=23f7fbc112f1353900c1181e3ed36707, REOPEN/MOVE 2024-11-14T12:54:34,226 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true 2024-11-14T12:54:34,226 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:34,226 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:34,226 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T12:54:34,226 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:34,226 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:34,226 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:34,226 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN 2024-11-14T12:54:34,226 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN 2024-11-14T12:54:34,226 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN 2024-11-14T12:54:34,226 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f16209fa220cf1b7a18b891e4fe53c6, ASSIGN 2024-11-14T12:54:34,226 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,226 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,227 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,227 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,227 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=669ef94e8b26,43475,1731588864980 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,227 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,227 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,227 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,227 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN 2024-11-14T12:54:34,227 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,227 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN 2024-11-14T12:54:34,227 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,227 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN 2024-11-14T12:54:34,227 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f16209fa220cf1b7a18b891e4fe53c6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,227 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45309 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T12:54:34,228 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f16209fa220cf1b7a18b891e4fe53c6, ASSIGN 2024-11-14T12:54:34,228 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-14T12:54:34,229 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=65 2024-11-14T12:54:34,229 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=669ef94e8b26,43475,1731588864980, table=hbase:meta, region=1588230740 2024-11-14T12:54:34,229 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 669ef94e8b26,43475,1731588864980, state=OPEN 2024-11-14T12:54:34,238 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45309-0x1013ab2e72b0000, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:34,238 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43475-0x1013ab2e72b0001, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:34,238 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46877-0x1013ab2e72b0002, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:34,238 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:34,238 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:34,239 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:34,239 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN to state=OFFLINE, location=null, table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7 to restore RIT 2024-11-14T12:54:34,239 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN to state=OFFLINE, location=null, table=test, region=a572ae08c8738aa198c40f32aa4b5f96 to restore RIT 2024-11-14T12:54:34,239 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN to state=OFFLINE, location=null, table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8 to restore RIT 2024-11-14T12:54:34,239 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=4f16209fa220cf1b7a18b891e4fe53c6, ASSIGN to state=OFFLINE, location=null, table=test, region=4f16209fa220cf1b7a18b891e4fe53c6 to restore RIT 2024-11-14T12:54:34,240 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731588904240 2024-11-14T12:54:34,240 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-14T12:54:34,240 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-14T12:54:34,241 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-14T12:54:34,244 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-14T12:54:34,244 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN 2024-11-14T12:54:34,244 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T12:54:34,244 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-14T12:54:34,248 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a572ae08c8738aa198c40f32aa4b5f96, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,248 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2ccfc8c17ab1571bc4ea6d526017f9f, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,249 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd0512a8a2f36befd3453e1140df3af, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,249 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b8044551b940cf51bfa7c7633c4df21d, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,249 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f7fbc112f1353900c1181e3ed36707, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=5 2024-11-14T12:54:34,249 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf47be17fd22bc204db28da07b30af2c, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,249 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f76ec25f51f69b1106342a7dfae65fb6, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,249 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cdc4b47ed9fe07b56101f9e0e17abcd7, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,250 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03f289a2805f0b2f037581e9390ba191, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,250 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1358fa84af33d1799bd11ba7f4959a37, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,250 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=333a3eba2bb2d318c041beb4642f5ef3, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,250 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64b5bd4e6112417a55e871b36551c5b8, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,250 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e90cc230c4f13fece446e3e8420dddbe, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,250 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f16209fa220cf1b7a18b891e4fe53c6, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,250 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=338b47e35e9a8b9dd3045d365aaf47bd, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,251 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ea6e95809b33f853859fa227179bfe85, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,251 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=18ddd925c95b1b23296976da04f97e83, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,251 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5d884ba0737bc572475e022a6ed1bc8, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,251 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d33b9e9c25fcfc654d257646677381c, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,251 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8ddc4f331afb3a3007ef5c49cdf6ab8, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,251 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8c0d62856832eec9f9ca20bb6db602c, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,251 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87eb074dbd46b3bc6e27fb0bf3291d5b, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,251 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22b9f10374fc1ea7a876d476ce0b083d, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,251 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cadcf21958eaec53ee489ca0918e3ab7, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66b34b9be40614bd68d0cea995e44193, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2201f29440b3aa3259741bbb374e209d, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,252 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-14T12:54:34,252 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731588934252 2024-11-14T12:54:34,252 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731588994252 2024-11-14T12:54:34,252 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 11 msec 2024-11-14T12:54:34,252 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-14T12:54:34,252 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T12:54:34,252 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T12:54:34,252 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T12:54:34,252 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-14T12:54:34,253 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-14T12:54:34,253 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-14T12:54:34,253 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T12:54:34,254 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-14T12:54:34,256 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-14T12:54:34,256 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T12:54:34,256 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T12:54:34,256 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,256 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,256 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,256 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T12:54:34,256 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T12:54:34,256 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,256 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T12:54:34,256 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T12:54:34,257 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-14T12:54:34,257 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=23f7fbc112f1353900c1181e3ed36707, REOPEN/MOVE 2024-11-14T12:54:34,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true 2024-11-14T12:54:34,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:34,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:34,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T12:54:34,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:34,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:34,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:34,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN 2024-11-14T12:54:34,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN 2024-11-14T12:54:34,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN 2024-11-14T12:54:34,258 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,258 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,258 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,258 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,258 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=669ef94e8b26,43475,1731588864980 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,259 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,259 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,259 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,259 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN 2024-11-14T12:54:34,259 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,259 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN 2024-11-14T12:54:34,259 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,259 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN 2024-11-14T12:54:34,259 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-14T12:54:34,260 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=65 2024-11-14T12:54:34,260 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=669ef94e8b26,43475,1731588864980, table=hbase:meta, region=1588230740 2024-11-14T12:54:34,260 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 669ef94e8b26,43475,1731588864980, state=OPEN 2024-11-14T12:54:34,270 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46877-0x1013ab2e72b0002, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:34,270 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45309-0x1013ab2e72b0000, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:34,270 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43475-0x1013ab2e72b0001, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:34,270 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:34,270 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:34,270 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:34,270 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN to state=OFFLINE, location=null, table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7 to restore RIT 2024-11-14T12:54:34,271 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN to state=OFFLINE, location=null, table=test, region=a572ae08c8738aa198c40f32aa4b5f96 to restore RIT 2024-11-14T12:54:34,271 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8, ASSIGN to state=OFFLINE, location=null, table=test, region=c8ddc4f331afb3a3007ef5c49cdf6ab8 to restore RIT 2024-11-14T12:54:34,271 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731588904271 2024-11-14T12:54:34,271 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-14T12:54:34,271 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-14T12:54:34,273 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-14T12:54:34,275 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-14T12:54:34,275 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN 2024-11-14T12:54:34,275 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T12:54:34,275 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-14T12:54:34,277 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a572ae08c8738aa198c40f32aa4b5f96, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,277 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2ccfc8c17ab1571bc4ea6d526017f9f, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,278 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd0512a8a2f36befd3453e1140df3af, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,278 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b8044551b940cf51bfa7c7633c4df21d, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,278 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f7fbc112f1353900c1181e3ed36707, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=5 2024-11-14T12:54:34,278 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf47be17fd22bc204db28da07b30af2c, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,278 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f76ec25f51f69b1106342a7dfae65fb6, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,278 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cdc4b47ed9fe07b56101f9e0e17abcd7, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,279 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03f289a2805f0b2f037581e9390ba191, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,279 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1358fa84af33d1799bd11ba7f4959a37, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,279 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=333a3eba2bb2d318c041beb4642f5ef3, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,279 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64b5bd4e6112417a55e871b36551c5b8, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,279 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e90cc230c4f13fece446e3e8420dddbe, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,279 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f16209fa220cf1b7a18b891e4fe53c6, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,279 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=338b47e35e9a8b9dd3045d365aaf47bd, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,279 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ea6e95809b33f853859fa227179bfe85, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,279 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=18ddd925c95b1b23296976da04f97e83, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,280 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5d884ba0737bc572475e022a6ed1bc8, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,280 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d33b9e9c25fcfc654d257646677381c, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,280 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8ddc4f331afb3a3007ef5c49cdf6ab8, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,280 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8c0d62856832eec9f9ca20bb6db602c, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,280 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87eb074dbd46b3bc6e27fb0bf3291d5b, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,280 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22b9f10374fc1ea7a876d476ce0b083d, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,280 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cadcf21958eaec53ee489ca0918e3ab7, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,280 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66b34b9be40614bd68d0cea995e44193, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,281 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2201f29440b3aa3259741bbb374e209d, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,281 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-14T12:54:34,281 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731588934281 2024-11-14T12:54:34,281 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731588994281 2024-11-14T12:54:34,281 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-14T12:54:34,281 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-14T12:54:34,281 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T12:54:34,281 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T12:54:34,281 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T12:54:34,281 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-14T12:54:34,281 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-14T12:54:34,281 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-14T12:54:34,282 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T12:54:34,282 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-14T12:54:34,284 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-14T12:54:34,284 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T12:54:34,284 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T12:54:34,284 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,284 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,284 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T12:54:34,284 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T12:54:34,284 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,284 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T12:54:34,284 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T12:54:34,285 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-14T12:54:34,285 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=23f7fbc112f1353900c1181e3ed36707, REOPEN/MOVE 2024-11-14T12:54:34,286 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true 2024-11-14T12:54:34,286 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:34,286 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:34,286 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T12:54:34,286 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:34,286 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:34,286 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:34,286 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN 2024-11-14T12:54:34,286 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN 2024-11-14T12:54:34,286 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,286 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,287 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,287 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,287 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=669ef94e8b26,43475,1731588864980 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,287 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,287 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,287 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,287 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN 2024-11-14T12:54:34,287 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,287 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN 2024-11-14T12:54:34,287 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-14T12:54:34,288 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=65 2024-11-14T12:54:34,288 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=669ef94e8b26,43475,1731588864980, table=hbase:meta, region=1588230740 2024-11-14T12:54:34,289 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 669ef94e8b26,43475,1731588864980, state=OPEN 2024-11-14T12:54:34,331 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45309 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T12:54:34,333 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46877-0x1013ab2e72b0002, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:34,333 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43475-0x1013ab2e72b0001, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:34,333 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:34,333 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:34,333 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45309-0x1013ab2e72b0000, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:34,333 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:34,334 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN to state=OFFLINE, location=null, table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7 to restore RIT 2024-11-14T12:54:34,334 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a572ae08c8738aa198c40f32aa4b5f96, ASSIGN to state=OFFLINE, location=null, table=test, region=a572ae08c8738aa198c40f32aa4b5f96 to restore RIT 2024-11-14T12:54:34,335 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731588904335 2024-11-14T12:54:34,335 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-14T12:54:34,335 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-14T12:54:34,336 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-14T12:54:34,337 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-14T12:54:34,337 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN 2024-11-14T12:54:34,337 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T12:54:34,337 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-14T12:54:34,341 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a572ae08c8738aa198c40f32aa4b5f96, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,342 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2ccfc8c17ab1571bc4ea6d526017f9f, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,342 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd0512a8a2f36befd3453e1140df3af, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,342 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b8044551b940cf51bfa7c7633c4df21d, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,342 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f7fbc112f1353900c1181e3ed36707, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=5 2024-11-14T12:54:34,342 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf47be17fd22bc204db28da07b30af2c, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,342 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f76ec25f51f69b1106342a7dfae65fb6, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,343 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cdc4b47ed9fe07b56101f9e0e17abcd7, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,343 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03f289a2805f0b2f037581e9390ba191, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,343 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1358fa84af33d1799bd11ba7f4959a37, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,343 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=333a3eba2bb2d318c041beb4642f5ef3, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,343 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64b5bd4e6112417a55e871b36551c5b8, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,343 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e90cc230c4f13fece446e3e8420dddbe, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,343 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f16209fa220cf1b7a18b891e4fe53c6, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,344 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=338b47e35e9a8b9dd3045d365aaf47bd, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,344 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ea6e95809b33f853859fa227179bfe85, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,344 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=18ddd925c95b1b23296976da04f97e83, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,344 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5d884ba0737bc572475e022a6ed1bc8, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,344 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d33b9e9c25fcfc654d257646677381c, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,344 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8ddc4f331afb3a3007ef5c49cdf6ab8, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,344 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8c0d62856832eec9f9ca20bb6db602c, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,344 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87eb074dbd46b3bc6e27fb0bf3291d5b, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,345 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22b9f10374fc1ea7a876d476ce0b083d, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,345 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cadcf21958eaec53ee489ca0918e3ab7, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,345 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66b34b9be40614bd68d0cea995e44193, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,345 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2201f29440b3aa3259741bbb374e209d, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,345 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-14T12:54:34,345 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731588934345 2024-11-14T12:54:34,345 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731588994345 2024-11-14T12:54:34,345 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 10 msec 2024-11-14T12:54:34,345 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-14T12:54:34,346 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T12:54:34,346 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T12:54:34,346 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T12:54:34,346 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-14T12:54:34,346 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-14T12:54:34,346 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-14T12:54:34,346 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T12:54:34,347 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-14T12:54:34,349 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-14T12:54:34,349 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T12:54:34,349 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T12:54:34,349 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45309 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T12:54:34,350 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,350 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T12:54:34,350 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T12:54:34,350 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,350 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T12:54:34,350 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T12:54:34,351 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-14T12:54:34,351 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=23f7fbc112f1353900c1181e3ed36707, REOPEN/MOVE 2024-11-14T12:54:34,352 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true 2024-11-14T12:54:34,352 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:34,353 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:34,353 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T12:54:34,353 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:34,353 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:34,353 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:34,353 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN 2024-11-14T12:54:34,353 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,353 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,353 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,353 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,353 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=669ef94e8b26,43475,1731588864980 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,353 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,353 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,353 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,353 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN 2024-11-14T12:54:34,354 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-14T12:54:34,356 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=65 2024-11-14T12:54:34,356 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=669ef94e8b26,43475,1731588864980, table=hbase:meta, region=1588230740 2024-11-14T12:54:34,356 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 669ef94e8b26,43475,1731588864980, state=OPEN 2024-11-14T12:54:34,372 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46877-0x1013ab2e72b0002, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:34,372 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43475-0x1013ab2e72b0001, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:34,372 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45309-0x1013ab2e72b0000, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:34,372 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:34,372 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:34,372 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:34,372 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7, ASSIGN to state=OFFLINE, location=null, table=test, region=cdc4b47ed9fe07b56101f9e0e17abcd7 to restore RIT 2024-11-14T12:54:34,374 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731588904374 2024-11-14T12:54:34,374 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-14T12:54:34,374 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-14T12:54:34,376 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-14T12:54:34,378 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-14T12:54:34,378 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:34,378 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T12:54:34,378 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-14T12:54:34,381 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a572ae08c8738aa198c40f32aa4b5f96, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,381 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2ccfc8c17ab1571bc4ea6d526017f9f, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,381 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd0512a8a2f36befd3453e1140df3af, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,381 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b8044551b940cf51bfa7c7633c4df21d, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,381 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f7fbc112f1353900c1181e3ed36707, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=5 2024-11-14T12:54:34,381 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf47be17fd22bc204db28da07b30af2c, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,382 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f76ec25f51f69b1106342a7dfae65fb6, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,382 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cdc4b47ed9fe07b56101f9e0e17abcd7, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,382 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03f289a2805f0b2f037581e9390ba191, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,382 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1358fa84af33d1799bd11ba7f4959a37, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,382 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=333a3eba2bb2d318c041beb4642f5ef3, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,382 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64b5bd4e6112417a55e871b36551c5b8, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,382 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e90cc230c4f13fece446e3e8420dddbe, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,383 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f16209fa220cf1b7a18b891e4fe53c6, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,383 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=338b47e35e9a8b9dd3045d365aaf47bd, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,383 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ea6e95809b33f853859fa227179bfe85, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,383 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=18ddd925c95b1b23296976da04f97e83, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,383 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5d884ba0737bc572475e022a6ed1bc8, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,383 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d33b9e9c25fcfc654d257646677381c, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,383 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8ddc4f331afb3a3007ef5c49cdf6ab8, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,383 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8c0d62856832eec9f9ca20bb6db602c, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,383 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87eb074dbd46b3bc6e27fb0bf3291d5b, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,384 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22b9f10374fc1ea7a876d476ce0b083d, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,384 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cadcf21958eaec53ee489ca0918e3ab7, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,384 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66b34b9be40614bd68d0cea995e44193, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,384 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2201f29440b3aa3259741bbb374e209d, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,384 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-14T12:54:34,384 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731588934384 2024-11-14T12:54:34,384 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731588994384 2024-11-14T12:54:34,384 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 10 msec 2024-11-14T12:54:34,384 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-14T12:54:34,384 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T12:54:34,384 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T12:54:34,384 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T12:54:34,385 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-14T12:54:34,385 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-14T12:54:34,385 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-14T12:54:34,385 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T12:54:34,386 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-14T12:54:34,388 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-14T12:54:34,388 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T12:54:34,388 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T12:54:34,388 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T12:54:34,388 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T12:54:34,388 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,389 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T12:54:34,389 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T12:54:34,389 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-14T12:54:34,390 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=23f7fbc112f1353900c1181e3ed36707, REOPEN/MOVE 2024-11-14T12:54:34,390 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true 2024-11-14T12:54:34,391 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:34,391 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:34,391 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T12:54:34,391 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:34,391 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:34,391 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:34,391 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,391 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,391 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,391 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,391 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=669ef94e8b26,43475,1731588864980 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,391 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,391 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,391 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-14T12:54:34,392 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=65 2024-11-14T12:54:34,392 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=669ef94e8b26,43475,1731588864980, table=hbase:meta, region=1588230740 2024-11-14T12:54:34,393 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 669ef94e8b26,43475,1731588864980, state=OPEN 2024-11-14T12:54:34,403 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45309-0x1013ab2e72b0000, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:34,403 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46877-0x1013ab2e72b0002, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:34,403 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43475-0x1013ab2e72b0001, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:34,403 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:34,403 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:34,403 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:34,404 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731588904404 2024-11-14T12:54:34,404 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-14T12:54:34,404 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-14T12:54:34,405 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-14T12:54:34,407 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-14T12:54:34,407 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:34,407 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T12:54:34,407 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-14T12:54:34,410 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a572ae08c8738aa198c40f32aa4b5f96, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,410 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2ccfc8c17ab1571bc4ea6d526017f9f, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,410 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd0512a8a2f36befd3453e1140df3af, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,410 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b8044551b940cf51bfa7c7633c4df21d, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,410 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f7fbc112f1353900c1181e3ed36707, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=5 2024-11-14T12:54:34,410 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf47be17fd22bc204db28da07b30af2c, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,411 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f76ec25f51f69b1106342a7dfae65fb6, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,411 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cdc4b47ed9fe07b56101f9e0e17abcd7, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,411 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03f289a2805f0b2f037581e9390ba191, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,411 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1358fa84af33d1799bd11ba7f4959a37, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,411 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=333a3eba2bb2d318c041beb4642f5ef3, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,411 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64b5bd4e6112417a55e871b36551c5b8, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,411 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e90cc230c4f13fece446e3e8420dddbe, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,411 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f16209fa220cf1b7a18b891e4fe53c6, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,412 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=338b47e35e9a8b9dd3045d365aaf47bd, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,412 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ea6e95809b33f853859fa227179bfe85, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,412 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=18ddd925c95b1b23296976da04f97e83, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,412 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5d884ba0737bc572475e022a6ed1bc8, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,412 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d33b9e9c25fcfc654d257646677381c, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,412 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8ddc4f331afb3a3007ef5c49cdf6ab8, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,412 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8c0d62856832eec9f9ca20bb6db602c, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,412 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87eb074dbd46b3bc6e27fb0bf3291d5b, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,413 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22b9f10374fc1ea7a876d476ce0b083d, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,413 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cadcf21958eaec53ee489ca0918e3ab7, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,413 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66b34b9be40614bd68d0cea995e44193, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,413 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2201f29440b3aa3259741bbb374e209d, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,413 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-14T12:54:34,413 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731588934413 2024-11-14T12:54:34,413 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731588994413 2024-11-14T12:54:34,413 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-14T12:54:34,413 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-14T12:54:34,413 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T12:54:34,414 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T12:54:34,414 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T12:54:34,414 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-14T12:54:34,414 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-14T12:54:34,414 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-14T12:54:34,414 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T12:54:34,415 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-14T12:54:34,417 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-14T12:54:34,417 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T12:54:34,417 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T12:54:34,417 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T12:54:34,417 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,417 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T12:54:34,417 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T12:54:34,417 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-14T12:54:34,418 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=23f7fbc112f1353900c1181e3ed36707, REOPEN/MOVE 2024-11-14T12:54:34,418 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true 2024-11-14T12:54:34,418 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:34,418 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:34,418 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T12:54:34,418 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:34,418 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:34,418 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,418 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,419 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,419 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,419 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=669ef94e8b26,43475,1731588864980 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,419 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.1731588867078, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,419 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-14T12:54:34,420 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=65 2024-11-14T12:54:34,420 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=669ef94e8b26,43475,1731588864980, table=hbase:meta, region=1588230740 2024-11-14T12:54:34,420 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 669ef94e8b26,43475,1731588864980, state=OPEN 2024-11-14T12:54:34,435 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45309 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T12:54:34,453 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45309 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T12:54:34,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46877-0x1013ab2e72b0002, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:34,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43475-0x1013ab2e72b0001, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:34,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45309-0x1013ab2e72b0000, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:34,527 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:34,527 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:34,527 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:34,528 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731588904528 2024-11-14T12:54:34,528 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-14T12:54:34,528 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-14T12:54:34,529 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-14T12:54:34,530 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-14T12:54:34,531 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:34,531 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T12:54:34,531 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=669ef94e8b26,43475,1731588864980 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-14T12:54:34,534 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a572ae08c8738aa198c40f32aa4b5f96, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,534 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2ccfc8c17ab1571bc4ea6d526017f9f, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,534 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd0512a8a2f36befd3453e1140df3af, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,534 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b8044551b940cf51bfa7c7633c4df21d, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,535 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f7fbc112f1353900c1181e3ed36707, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=5 2024-11-14T12:54:34,535 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf47be17fd22bc204db28da07b30af2c, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,535 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f76ec25f51f69b1106342a7dfae65fb6, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,535 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cdc4b47ed9fe07b56101f9e0e17abcd7, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,535 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03f289a2805f0b2f037581e9390ba191, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,535 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1358fa84af33d1799bd11ba7f4959a37, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,535 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=333a3eba2bb2d318c041beb4642f5ef3, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,536 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64b5bd4e6112417a55e871b36551c5b8, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,536 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e90cc230c4f13fece446e3e8420dddbe, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,536 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f16209fa220cf1b7a18b891e4fe53c6, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,536 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=338b47e35e9a8b9dd3045d365aaf47bd, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,536 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ea6e95809b33f853859fa227179bfe85, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,536 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=18ddd925c95b1b23296976da04f97e83, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,536 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5d884ba0737bc572475e022a6ed1bc8, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,536 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d33b9e9c25fcfc654d257646677381c, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,536 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8ddc4f331afb3a3007ef5c49cdf6ab8, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,537 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8c0d62856832eec9f9ca20bb6db602c, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,537 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87eb074dbd46b3bc6e27fb0bf3291d5b, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,537 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22b9f10374fc1ea7a876d476ce0b083d, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,537 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cadcf21958eaec53ee489ca0918e3ab7, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,537 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66b34b9be40614bd68d0cea995e44193, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,537 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2201f29440b3aa3259741bbb374e209d, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,537 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-14T12:54:34,537 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731588934537 2024-11-14T12:54:34,537 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731588994537 2024-11-14T12:54:34,538 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-14T12:54:34,538 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-14T12:54:34,538 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T12:54:34,538 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T12:54:34,538 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T12:54:34,538 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-14T12:54:34,538 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-14T12:54:34,538 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-14T12:54:34,538 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T12:54:34,539 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-14T12:54:34,541 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-14T12:54:34,541 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T12:54:34,541 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T12:54:34,541 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,541 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-14T12:54:34,541 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T12:54:34,542 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-14T12:54:34,542 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-14T12:54:34,542 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=23f7fbc112f1353900c1181e3ed36707, REOPEN/MOVE 2024-11-14T12:54:34,542 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-14T12:54:34,542 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true 2024-11-14T12:54:34,542 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:34,542 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:34,542 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T12:54:34,543 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=669ef94e8b26,43475,1731588864980 2024-11-14T12:54:34,543 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,543 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,543 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,543 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,543 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=669ef94e8b26,43475,1731588864980 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,543 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-14T12:54:34,543 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test 2024-11-14T12:54:34,544 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test Metrics about Tables on a single HBase RegionServer 2024-11-14T12:54:34,544 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=65 2024-11-14T12:54:34,544 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=669ef94e8b26,43475,1731588864980, table=hbase:meta, region=1588230740 2024-11-14T12:54:34,544 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 669ef94e8b26,43475,1731588864980, state=OPEN 2024-11-14T12:54:34,544 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T12:54:34,544 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-14T12:54:34,545 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-14T12:54:34,545 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-14T12:54:34,563 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43475-0x1013ab2e72b0001, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:34,563 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45309-0x1013ab2e72b0000, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:34,563 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46877-0x1013ab2e72b0002, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:34,563 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:34,563 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:34,563 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:34,565 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731588904565 2024-11-14T12:54:34,565 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-14T12:54:34,565 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-14T12:54:34,567 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-14T12:54:34,568 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-14T12:54:34,568 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T12:54:34,568 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T12:54:34,568 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-14T12:54:34,571 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a572ae08c8738aa198c40f32aa4b5f96, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,571 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2ccfc8c17ab1571bc4ea6d526017f9f, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,571 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd0512a8a2f36befd3453e1140df3af, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,571 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b8044551b940cf51bfa7c7633c4df21d, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,571 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f7fbc112f1353900c1181e3ed36707, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=5 2024-11-14T12:54:34,571 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf47be17fd22bc204db28da07b30af2c, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,572 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f76ec25f51f69b1106342a7dfae65fb6, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,572 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cdc4b47ed9fe07b56101f9e0e17abcd7, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,572 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03f289a2805f0b2f037581e9390ba191, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,572 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1358fa84af33d1799bd11ba7f4959a37, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,572 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=333a3eba2bb2d318c041beb4642f5ef3, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,572 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64b5bd4e6112417a55e871b36551c5b8, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,572 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e90cc230c4f13fece446e3e8420dddbe, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,572 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f16209fa220cf1b7a18b891e4fe53c6, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,572 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=338b47e35e9a8b9dd3045d365aaf47bd, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,572 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ea6e95809b33f853859fa227179bfe85, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,572 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=18ddd925c95b1b23296976da04f97e83, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,572 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5d884ba0737bc572475e022a6ed1bc8, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,573 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d33b9e9c25fcfc654d257646677381c, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,573 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8ddc4f331afb3a3007ef5c49cdf6ab8, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,573 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8c0d62856832eec9f9ca20bb6db602c, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,573 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87eb074dbd46b3bc6e27fb0bf3291d5b, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,573 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22b9f10374fc1ea7a876d476ce0b083d, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,573 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cadcf21958eaec53ee489ca0918e3ab7, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,573 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66b34b9be40614bd68d0cea995e44193, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,573 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2201f29440b3aa3259741bbb374e209d, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,573 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-14T12:54:34,573 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731588934573 2024-11-14T12:54:34,573 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731588994573 2024-11-14T12:54:34,573 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-14T12:54:34,573 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-14T12:54:34,573 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T12:54:34,574 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T12:54:34,574 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T12:54:34,574 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-14T12:54:34,574 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-14T12:54:34,574 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-14T12:54:34,574 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T12:54:34,575 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-14T12:54:34,576 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-14T12:54:34,576 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T12:54:34,576 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T12:54:34,576 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-14T12:54:34,577 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T12:54:34,577 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-14T12:54:34,577 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=23f7fbc112f1353900c1181e3ed36707, REOPEN/MOVE 2024-11-14T12:54:34,578 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true 2024-11-14T12:54:34,578 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:34,578 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:34,578 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T12:54:34,578 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,578 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,578 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,578 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,578 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-14T12:54:34,579 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=65 2024-11-14T12:54:34,579 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=669ef94e8b26,43475,1731588864980, table=hbase:meta, region=1588230740 2024-11-14T12:54:34,579 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 669ef94e8b26,43475,1731588864980, state=OPEN 2024-11-14T12:54:34,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45309-0x1013ab2e72b0000, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:34,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43475-0x1013ab2e72b0001, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:34,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46877-0x1013ab2e72b0002, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:34,589 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:34,589 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:34,589 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:34,589 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731588904589 2024-11-14T12:54:34,589 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-14T12:54:34,589 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-14T12:54:34,590 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-14T12:54:34,591 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-14T12:54:34,591 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:34,591 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T12:54:34,591 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-14T12:54:34,595 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a572ae08c8738aa198c40f32aa4b5f96, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,595 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2ccfc8c17ab1571bc4ea6d526017f9f, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,595 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd0512a8a2f36befd3453e1140df3af, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,595 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b8044551b940cf51bfa7c7633c4df21d, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,595 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f7fbc112f1353900c1181e3ed36707, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=5 2024-11-14T12:54:34,595 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf47be17fd22bc204db28da07b30af2c, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,595 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f76ec25f51f69b1106342a7dfae65fb6, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,596 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cdc4b47ed9fe07b56101f9e0e17abcd7, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,596 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03f289a2805f0b2f037581e9390ba191, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,596 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1358fa84af33d1799bd11ba7f4959a37, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,596 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=333a3eba2bb2d318c041beb4642f5ef3, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,596 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64b5bd4e6112417a55e871b36551c5b8, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,596 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e90cc230c4f13fece446e3e8420dddbe, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,596 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f16209fa220cf1b7a18b891e4fe53c6, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,596 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=338b47e35e9a8b9dd3045d365aaf47bd, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,597 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ea6e95809b33f853859fa227179bfe85, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,597 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=18ddd925c95b1b23296976da04f97e83, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,597 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5d884ba0737bc572475e022a6ed1bc8, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,597 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d33b9e9c25fcfc654d257646677381c, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,597 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8ddc4f331afb3a3007ef5c49cdf6ab8, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,597 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8c0d62856832eec9f9ca20bb6db602c, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,597 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87eb074dbd46b3bc6e27fb0bf3291d5b, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,597 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22b9f10374fc1ea7a876d476ce0b083d, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,597 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cadcf21958eaec53ee489ca0918e3ab7, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,598 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66b34b9be40614bd68d0cea995e44193, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,598 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2201f29440b3aa3259741bbb374e209d, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,598 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-14T12:54:34,598 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731588934598 2024-11-14T12:54:34,598 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731588994598 2024-11-14T12:54:34,598 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-14T12:54:34,598 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-14T12:54:34,598 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T12:54:34,598 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T12:54:34,598 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T12:54:34,599 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-14T12:54:34,599 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-14T12:54:34,599 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-14T12:54:34,599 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T12:54:34,600 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-14T12:54:34,602 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-14T12:54:34,602 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T12:54:34,602 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-14T12:54:34,602 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T12:54:34,603 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-14T12:54:34,603 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=23f7fbc112f1353900c1181e3ed36707, REOPEN/MOVE 2024-11-14T12:54:34,604 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true 2024-11-14T12:54:34,604 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:34,604 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:34,604 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,604 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,604 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,604 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-14T12:54:34,605 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=65 2024-11-14T12:54:34,605 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=669ef94e8b26,43475,1731588864980, table=hbase:meta, region=1588230740 2024-11-14T12:54:34,605 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 669ef94e8b26,43475,1731588864980, state=OPEN 2024-11-14T12:54:34,613 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43475-0x1013ab2e72b0001, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:34,613 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46877-0x1013ab2e72b0002, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:34,613 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45309-0x1013ab2e72b0000, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:34,613 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:34,613 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:34,614 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:34,614 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731588904614 2024-11-14T12:54:34,614 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-14T12:54:34,615 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-14T12:54:34,616 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-14T12:54:34,617 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-14T12:54:34,617 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:34,618 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T12:54:34,618 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-14T12:54:34,620 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a572ae08c8738aa198c40f32aa4b5f96, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,620 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2ccfc8c17ab1571bc4ea6d526017f9f, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,620 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd0512a8a2f36befd3453e1140df3af, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,620 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b8044551b940cf51bfa7c7633c4df21d, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,620 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f7fbc112f1353900c1181e3ed36707, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=5 2024-11-14T12:54:34,620 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf47be17fd22bc204db28da07b30af2c, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,620 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f76ec25f51f69b1106342a7dfae65fb6, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,620 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cdc4b47ed9fe07b56101f9e0e17abcd7, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,620 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03f289a2805f0b2f037581e9390ba191, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,621 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1358fa84af33d1799bd11ba7f4959a37, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,621 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=333a3eba2bb2d318c041beb4642f5ef3, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,621 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64b5bd4e6112417a55e871b36551c5b8, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,621 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e90cc230c4f13fece446e3e8420dddbe, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,621 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f16209fa220cf1b7a18b891e4fe53c6, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,621 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=338b47e35e9a8b9dd3045d365aaf47bd, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,621 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ea6e95809b33f853859fa227179bfe85, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,621 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=18ddd925c95b1b23296976da04f97e83, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,621 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5d884ba0737bc572475e022a6ed1bc8, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,621 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d33b9e9c25fcfc654d257646677381c, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,621 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8ddc4f331afb3a3007ef5c49cdf6ab8, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,621 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8c0d62856832eec9f9ca20bb6db602c, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,622 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87eb074dbd46b3bc6e27fb0bf3291d5b, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,622 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22b9f10374fc1ea7a876d476ce0b083d, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,622 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cadcf21958eaec53ee489ca0918e3ab7, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,622 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66b34b9be40614bd68d0cea995e44193, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,622 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2201f29440b3aa3259741bbb374e209d, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,622 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-14T12:54:34,622 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731588934622 2024-11-14T12:54:34,622 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731588994622 2024-11-14T12:54:34,622 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 7 msec 2024-11-14T12:54:34,622 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-14T12:54:34,622 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T12:54:34,622 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T12:54:34,622 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T12:54:34,623 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-14T12:54:34,623 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-14T12:54:34,623 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-14T12:54:34,623 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T12:54:34,624 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-14T12:54:34,625 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-14T12:54:34,625 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-14T12:54:34,625 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T12:54:34,625 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-14T12:54:34,625 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=23f7fbc112f1353900c1181e3ed36707, REOPEN/MOVE 2024-11-14T12:54:34,626 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true 2024-11-14T12:54:34,626 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 2024-11-14T12:54:34,626 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,626 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 669ef94e8b26%2C44819%2C1731588865173.meta.1731588867504.meta, worker=669ef94e8b26,46877,1731588865107 didn't hold the lock before restarting, skip acquiring lock. 2024-11-14T12:54:34,626 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-14T12:54:34,627 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=65 2024-11-14T12:54:34,627 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=669ef94e8b26,43475,1731588864980, table=hbase:meta, region=1588230740 2024-11-14T12:54:34,627 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 669ef94e8b26,43475,1731588864980, state=OPEN 2024-11-14T12:54:34,628 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45309 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T12:54:34,629 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45309 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T12:54:34,636 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43475-0x1013ab2e72b0001, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:34,636 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45309-0x1013ab2e72b0000, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:34,636 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46877-0x1013ab2e72b0002, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:34,637 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:34,637 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:34,637 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:34,638 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731588904638 2024-11-14T12:54:34,638 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-14T12:54:34,638 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-14T12:54:34,639 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-14T12:54:34,640 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-14T12:54:34,640 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true 2024-11-14T12:54:34,640 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T12:54:34,640 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1604) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-14T12:54:34,643 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a572ae08c8738aa198c40f32aa4b5f96, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,643 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2ccfc8c17ab1571bc4ea6d526017f9f, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,643 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd0512a8a2f36befd3453e1140df3af, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,643 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b8044551b940cf51bfa7c7633c4df21d, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,643 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f7fbc112f1353900c1181e3ed36707, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=5 2024-11-14T12:54:34,643 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf47be17fd22bc204db28da07b30af2c, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,644 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f76ec25f51f69b1106342a7dfae65fb6, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,644 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cdc4b47ed9fe07b56101f9e0e17abcd7, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,644 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03f289a2805f0b2f037581e9390ba191, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,644 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1358fa84af33d1799bd11ba7f4959a37, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,644 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=333a3eba2bb2d318c041beb4642f5ef3, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,644 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64b5bd4e6112417a55e871b36551c5b8, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,644 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e90cc230c4f13fece446e3e8420dddbe, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,644 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f16209fa220cf1b7a18b891e4fe53c6, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,644 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=338b47e35e9a8b9dd3045d365aaf47bd, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,644 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ea6e95809b33f853859fa227179bfe85, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,645 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=18ddd925c95b1b23296976da04f97e83, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,645 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5d884ba0737bc572475e022a6ed1bc8, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,645 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d33b9e9c25fcfc654d257646677381c, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,645 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8ddc4f331afb3a3007ef5c49cdf6ab8, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,645 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8c0d62856832eec9f9ca20bb6db602c, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,645 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87eb074dbd46b3bc6e27fb0bf3291d5b, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,645 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22b9f10374fc1ea7a876d476ce0b083d, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,645 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cadcf21958eaec53ee489ca0918e3ab7, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,645 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66b34b9be40614bd68d0cea995e44193, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,646 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2201f29440b3aa3259741bbb374e209d, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,646 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-14T12:54:34,646 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731588934646 2024-11-14T12:54:34,646 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731588994646 2024-11-14T12:54:34,646 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-14T12:54:34,646 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-14T12:54:34,646 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T12:54:34,646 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T12:54:34,646 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T12:54:34,646 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-14T12:54:34,646 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-14T12:54:34,646 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-14T12:54:34,647 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T12:54:34,648 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-14T12:54:34,649 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-14T12:54:34,649 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T12:54:34,650 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-14T12:54:34,650 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=23f7fbc112f1353900c1181e3ed36707, REOPEN/MOVE 2024-11-14T12:54:34,651 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true 2024-11-14T12:54:34,651 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-14T12:54:34,651 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-14T12:54:34,651 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=65 2024-11-14T12:54:34,652 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=669ef94e8b26,43475,1731588864980, table=hbase:meta, region=1588230740 2024-11-14T12:54:34,652 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 669ef94e8b26,43475,1731588864980, state=OPEN 2024-11-14T12:54:34,661 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45309-0x1013ab2e72b0000, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:34,661 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43475-0x1013ab2e72b0001, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:34,662 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46877-0x1013ab2e72b0002, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T12:54:34,662 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:34,662 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:34,662 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T12:54:34,662 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731588904662 2024-11-14T12:54:34,663 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-14T12:54:34,663 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-14T12:54:34,664 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-14T12:54:34,666 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=60, state=ROLLEDBACK, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure 333a3eba2bb2d318c041beb4642f5ef3, server=669ef94e8b26,43475,1731588864980:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 669ef94e8b26,44819,1731588865173, splitWal=true, meta=true exec-time=2.9830 sec 2024-11-14T12:54:34,668 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a572ae08c8738aa198c40f32aa4b5f96, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,668 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2ccfc8c17ab1571bc4ea6d526017f9f, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,668 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fd0512a8a2f36befd3453e1140df3af, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,668 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b8044551b940cf51bfa7c7633c4df21d, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,668 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f7fbc112f1353900c1181e3ed36707, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=5 2024-11-14T12:54:34,668 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bf47be17fd22bc204db28da07b30af2c, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,668 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f76ec25f51f69b1106342a7dfae65fb6, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,668 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cdc4b47ed9fe07b56101f9e0e17abcd7, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,668 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03f289a2805f0b2f037581e9390ba191, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,668 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1358fa84af33d1799bd11ba7f4959a37, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=333a3eba2bb2d318c041beb4642f5ef3, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64b5bd4e6112417a55e871b36551c5b8, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e90cc230c4f13fece446e3e8420dddbe, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4f16209fa220cf1b7a18b891e4fe53c6, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=338b47e35e9a8b9dd3045d365aaf47bd, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ea6e95809b33f853859fa227179bfe85, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=18ddd925c95b1b23296976da04f97e83, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5d884ba0737bc572475e022a6ed1bc8, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d33b9e9c25fcfc654d257646677381c, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c8ddc4f331afb3a3007ef5c49cdf6ab8, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8c0d62856832eec9f9ca20bb6db602c, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87eb074dbd46b3bc6e27fb0bf3291d5b, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22b9f10374fc1ea7a876d476ce0b083d, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cadcf21958eaec53ee489ca0918e3ab7, regionState=OPEN, lastHost=669ef94e8b26,46877,1731588865107, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=66b34b9be40614bd68d0cea995e44193, regionState=OPEN, lastHost=669ef94e8b26,43475,1731588864980, regionLocation=669ef94e8b26,43475,1731588864980, openSeqNum=2 2024-11-14T12:54:34,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2201f29440b3aa3259741bbb374e209d, regionState=OPENING, lastHost=669ef94e8b26,44819,1731588865173, regionLocation=669ef94e8b26,46877,1731588865107, openSeqNum=2 2024-11-14T12:54:34,670 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-14T12:54:34,670 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731588934670 2024-11-14T12:54:34,670 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731588994670 2024-11-14T12:54:34,670 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 7 msec 2024-11-14T12:54:34,689 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.assignment.TestRollbackSCP#testFailAndRollback Thread=353 (was 387), OpenFileDescriptor=622 (was 645), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=258 (was 263), ProcessCount=11 (was 11), AvailableMemoryMB=7374 (was 7493) 2024-11-14T12:54:34,689 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-14T12:54:34,689 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T12:54:34,689 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T12:54:34,690 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T12:54:34,690 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T12:54:34,690 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T12:54:34,690 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-14T12:54:34,690 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1352866209, stopped=false 2024-11-14T12:54:34,691 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=669ef94e8b26,45309,1731588864185 2024-11-14T12:54:34,696 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43475-0x1013ab2e72b0001, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T12:54:34,696 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46877-0x1013ab2e72b0002, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T12:54:34,697 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45309-0x1013ab2e72b0000, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T12:54:34,697 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46877-0x1013ab2e72b0002, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T12:54:34,697 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45309-0x1013ab2e72b0000, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T12:54:34,697 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T12:54:34,697 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43475-0x1013ab2e72b0001, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T12:54:34,697 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T12:54:34,697 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T12:54:34,697 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T12:54:34,697 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43475-0x1013ab2e72b0001, quorum=127.0.0.1:60896, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T12:54:34,697 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '669ef94e8b26,43475,1731588864980' ***** 2024-11-14T12:54:34,697 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-14T12:54:34,697 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45309-0x1013ab2e72b0000, quorum=127.0.0.1:60896, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T12:54:34,698 INFO [RS:0;669ef94e8b26:43475 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-14T12:54:34,698 INFO [RS:0;669ef94e8b26:43475 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-14T12:54:34,698 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-14T12:54:34,698 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46877-0x1013ab2e72b0002, quorum=127.0.0.1:60896, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T12:54:34,698 INFO [RS:0;669ef94e8b26:43475 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-14T12:54:34,698 INFO [RS:0;669ef94e8b26:43475 {}] regionserver.HRegionServer(3091): Received CLOSE for 66b34b9be40614bd68d0cea995e44193 2024-11-14T12:54:34,698 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '669ef94e8b26,46877,1731588865107' ***** 2024-11-14T12:54:34,698 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-14T12:54:34,698 INFO [RS:1;669ef94e8b26:46877 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-14T12:54:34,698 INFO [RS:1;669ef94e8b26:46877 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-14T12:54:34,698 INFO [RS:1;669ef94e8b26:46877 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-14T12:54:34,698 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-14T12:54:34,698 INFO [RS:1;669ef94e8b26:46877 {}] regionserver.HRegionServer(3091): Received CLOSE for 0fd0512a8a2f36befd3453e1140df3af 2024-11-14T12:54:34,698 INFO [RS:0;669ef94e8b26:43475 {}] regionserver.HRegionServer(3091): Received CLOSE for a572ae08c8738aa198c40f32aa4b5f96 2024-11-14T12:54:34,699 INFO [RS:0;669ef94e8b26:43475 {}] regionserver.HRegionServer(3091): Received CLOSE for 64b5bd4e6112417a55e871b36551c5b8 2024-11-14T12:54:34,699 INFO [RS:0;669ef94e8b26:43475 {}] regionserver.HRegionServer(3091): Received CLOSE for b8044551b940cf51bfa7c7633c4df21d 2024-11-14T12:54:34,699 INFO [RS:0;669ef94e8b26:43475 {}] regionserver.HRegionServer(3091): Received CLOSE for bf47be17fd22bc204db28da07b30af2c 2024-11-14T12:54:34,699 INFO [RS:0;669ef94e8b26:43475 {}] regionserver.HRegionServer(3091): Received CLOSE for 338b47e35e9a8b9dd3045d365aaf47bd 2024-11-14T12:54:34,699 INFO [RS:0;669ef94e8b26:43475 {}] regionserver.HRegionServer(3091): Received CLOSE for 03f289a2805f0b2f037581e9390ba191 2024-11-14T12:54:34,699 INFO [RS:1;669ef94e8b26:46877 {}] regionserver.HRegionServer(3091): Received CLOSE for cadcf21958eaec53ee489ca0918e3ab7 2024-11-14T12:54:34,699 INFO [RS:0;669ef94e8b26:43475 {}] regionserver.HRegionServer(3091): Received CLOSE for 22b9f10374fc1ea7a876d476ce0b083d 2024-11-14T12:54:34,699 INFO [RS:1;669ef94e8b26:46877 {}] regionserver.HRegionServer(3091): Received CLOSE for 1358fa84af33d1799bd11ba7f4959a37 2024-11-14T12:54:34,699 INFO [RS:1;669ef94e8b26:46877 {}] regionserver.HRegionServer(3091): Received CLOSE for ea6e95809b33f853859fa227179bfe85 2024-11-14T12:54:34,699 INFO [RS:0;669ef94e8b26:43475 {}] regionserver.HRegionServer(3091): Received CLOSE for f8c0d62856832eec9f9ca20bb6db602c 2024-11-14T12:54:34,699 INFO [RS:1;669ef94e8b26:46877 {}] regionserver.HRegionServer(3091): Received CLOSE for f76ec25f51f69b1106342a7dfae65fb6 2024-11-14T12:54:34,699 INFO [RS:0;669ef94e8b26:43475 {}] regionserver.HRegionServer(3091): Received CLOSE for 333a3eba2bb2d318c041beb4642f5ef3 2024-11-14T12:54:34,699 INFO [RS:1;669ef94e8b26:46877 {}] regionserver.HRegionServer(3091): Received CLOSE for 87eb074dbd46b3bc6e27fb0bf3291d5b 2024-11-14T12:54:34,699 INFO [RS:1;669ef94e8b26:46877 {}] regionserver.HRegionServer(3091): Received CLOSE for 23f7fbc112f1353900c1181e3ed36707 2024-11-14T12:54:34,699 INFO [RS:0;669ef94e8b26:43475 {}] regionserver.HRegionServer(3091): Received CLOSE for a5d884ba0737bc572475e022a6ed1bc8 2024-11-14T12:54:34,699 INFO [RS:1;669ef94e8b26:46877 {}] regionserver.HRegionServer(3091): Received CLOSE for 1d33b9e9c25fcfc654d257646677381c 2024-11-14T12:54:34,699 INFO [RS:0;669ef94e8b26:43475 {}] regionserver.HRegionServer(3091): Received CLOSE for c2ccfc8c17ab1571bc4ea6d526017f9f 2024-11-14T12:54:34,699 INFO [RS:1;669ef94e8b26:46877 {}] regionserver.HRegionServer(3091): Received CLOSE for e90cc230c4f13fece446e3e8420dddbe 2024-11-14T12:54:34,699 INFO [RS:0;669ef94e8b26:43475 {}] regionserver.HRegionServer(959): stopping server 669ef94e8b26,43475,1731588864980 2024-11-14T12:54:34,699 INFO [RS:1;669ef94e8b26:46877 {}] regionserver.HRegionServer(959): stopping server 669ef94e8b26,46877,1731588865107 2024-11-14T12:54:34,699 INFO [RS:0;669ef94e8b26:43475 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T12:54:34,699 INFO [RS:1;669ef94e8b26:46877 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T12:54:34,699 INFO [RS:0;669ef94e8b26:43475 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;669ef94e8b26:43475. 2024-11-14T12:54:34,699 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 66b34b9be40614bd68d0cea995e44193, disabling compactions & flushes 2024-11-14T12:54:34,699 INFO [RS:1;669ef94e8b26:46877 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;669ef94e8b26:46877. 2024-11-14T12:54:34,699 DEBUG [RS:0;669ef94e8b26:43475 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T12:54:34,699 INFO [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,yyy,1731588868017.66b34b9be40614bd68d0cea995e44193. 2024-11-14T12:54:34,700 DEBUG [RS:1;669ef94e8b26:46877 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T12:54:34,700 DEBUG [RS:0;669ef94e8b26:43475 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T12:54:34,700 DEBUG [RS:1;669ef94e8b26:46877 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T12:54:34,700 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1731588868017.66b34b9be40614bd68d0cea995e44193. 2024-11-14T12:54:34,700 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0fd0512a8a2f36befd3453e1140df3af, disabling compactions & flushes 2024-11-14T12:54:34,700 INFO [RS:1;669ef94e8b26:46877 {}] regionserver.HRegionServer(1321): Waiting on 9 regions to close 2024-11-14T12:54:34,700 INFO [RS:0;669ef94e8b26:43475 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-14T12:54:34,700 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1731588868017.66b34b9be40614bd68d0cea995e44193. after waiting 0 ms 2024-11-14T12:54:34,700 INFO [RS:0;669ef94e8b26:43475 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-14T12:54:34,700 INFO [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ccc,1731588868017.0fd0512a8a2f36befd3453e1140df3af. 2024-11-14T12:54:34,700 DEBUG [RS:1;669ef94e8b26:46877 {}] regionserver.HRegionServer(1325): Online Regions={0fd0512a8a2f36befd3453e1140df3af=test,ccc,1731588868017.0fd0512a8a2f36befd3453e1140df3af., cadcf21958eaec53ee489ca0918e3ab7=test,xxx,1731588868017.cadcf21958eaec53ee489ca0918e3ab7., 1358fa84af33d1799bd11ba7f4959a37=test,jjj,1731588868017.1358fa84af33d1799bd11ba7f4959a37., ea6e95809b33f853859fa227179bfe85=test,ppp,1731588868017.ea6e95809b33f853859fa227179bfe85., f76ec25f51f69b1106342a7dfae65fb6=test,ggg,1731588868017.f76ec25f51f69b1106342a7dfae65fb6., 87eb074dbd46b3bc6e27fb0bf3291d5b=test,vvv,1731588868017.87eb074dbd46b3bc6e27fb0bf3291d5b., 23f7fbc112f1353900c1181e3ed36707=test,eee,1731588868017.23f7fbc112f1353900c1181e3ed36707., 1d33b9e9c25fcfc654d257646677381c=test,sss,1731588868017.1d33b9e9c25fcfc654d257646677381c., e90cc230c4f13fece446e3e8420dddbe=test,mmm,1731588868017.e90cc230c4f13fece446e3e8420dddbe.} 2024-11-14T12:54:34,700 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1731588868017.66b34b9be40614bd68d0cea995e44193. 2024-11-14T12:54:34,700 INFO [RS:0;669ef94e8b26:43475 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-14T12:54:34,700 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1731588868017.0fd0512a8a2f36befd3453e1140df3af. 2024-11-14T12:54:34,700 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1731588868017.0fd0512a8a2f36befd3453e1140df3af. after waiting 0 ms 2024-11-14T12:54:34,700 INFO [RS:0;669ef94e8b26:43475 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-14T12:54:34,700 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1731588868017.0fd0512a8a2f36befd3453e1140df3af. 2024-11-14T12:54:34,700 DEBUG [RS:1;669ef94e8b26:46877 {}] regionserver.HRegionServer(1351): Waiting on 0fd0512a8a2f36befd3453e1140df3af, 1358fa84af33d1799bd11ba7f4959a37, 1d33b9e9c25fcfc654d257646677381c, 2201f29440b3aa3259741bbb374e209d, 23f7fbc112f1353900c1181e3ed36707, 87eb074dbd46b3bc6e27fb0bf3291d5b, cadcf21958eaec53ee489ca0918e3ab7, e90cc230c4f13fece446e3e8420dddbe, ea6e95809b33f853859fa227179bfe85, f76ec25f51f69b1106342a7dfae65fb6 2024-11-14T12:54:34,700 INFO [RS:0;669ef94e8b26:43475 {}] regionserver.HRegionServer(1321): Waiting on 13 regions to close 2024-11-14T12:54:34,701 DEBUG [RS:0;669ef94e8b26:43475 {}] regionserver.HRegionServer(1351): Waiting on 03f289a2805f0b2f037581e9390ba191, 1588230740, 18ddd925c95b1b23296976da04f97e83, 22b9f10374fc1ea7a876d476ce0b083d, 333a3eba2bb2d318c041beb4642f5ef3, 338b47e35e9a8b9dd3045d365aaf47bd, 64b5bd4e6112417a55e871b36551c5b8, 66b34b9be40614bd68d0cea995e44193, a572ae08c8738aa198c40f32aa4b5f96, a5d884ba0737bc572475e022a6ed1bc8, b8044551b940cf51bfa7c7633c4df21d, bf47be17fd22bc204db28da07b30af2c, c2ccfc8c17ab1571bc4ea6d526017f9f, f8c0d62856832eec9f9ca20bb6db602c 2024-11-14T12:54:34,701 DEBUG [RS_CLOSE_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T12:54:34,701 INFO [RS_CLOSE_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T12:54:34,701 DEBUG [RS_CLOSE_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T12:54:34,701 DEBUG [RS_CLOSE_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T12:54:34,701 DEBUG [RS_CLOSE_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T12:54:34,701 INFO [RS_CLOSE_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.64 KB heapSize=5.42 KB 2024-11-14T12:54:34,704 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/0fd0512a8a2f36befd3453e1140df3af/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T12:54:34,705 INFO [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ccc,1731588868017.0fd0512a8a2f36befd3453e1140df3af. 2024-11-14T12:54:34,705 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0fd0512a8a2f36befd3453e1140df3af: Waiting for close lock at 1731588874700Running coprocessor pre-close hooks at 1731588874700Disabling compacts and flushes for region at 1731588874700Disabling writes for close at 1731588874700Writing region close event to WAL at 1731588874701 (+1 ms)Running coprocessor post-close hooks at 1731588874705 (+4 ms)Closed at 1731588874705 2024-11-14T12:54:34,705 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ccc,1731588868017.0fd0512a8a2f36befd3453e1140df3af. 2024-11-14T12:54:34,705 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing cadcf21958eaec53ee489ca0918e3ab7, disabling compactions & flushes 2024-11-14T12:54:34,705 INFO [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,xxx,1731588868017.cadcf21958eaec53ee489ca0918e3ab7. 2024-11-14T12:54:34,705 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1731588868017.cadcf21958eaec53ee489ca0918e3ab7. 2024-11-14T12:54:34,705 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1731588868017.cadcf21958eaec53ee489ca0918e3ab7. after waiting 0 ms 2024-11-14T12:54:34,705 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1731588868017.cadcf21958eaec53ee489ca0918e3ab7. 2024-11-14T12:54:34,707 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/66b34b9be40614bd68d0cea995e44193/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T12:54:34,708 INFO [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,yyy,1731588868017.66b34b9be40614bd68d0cea995e44193. 2024-11-14T12:54:34,708 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 66b34b9be40614bd68d0cea995e44193: Waiting for close lock at 1731588874699Running coprocessor pre-close hooks at 1731588874699Disabling compacts and flushes for region at 1731588874699Disabling writes for close at 1731588874700 (+1 ms)Writing region close event to WAL at 1731588874700Running coprocessor post-close hooks at 1731588874708 (+8 ms)Closed at 1731588874708 2024-11-14T12:54:34,708 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,yyy,1731588868017.66b34b9be40614bd68d0cea995e44193. 2024-11-14T12:54:34,709 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a572ae08c8738aa198c40f32aa4b5f96, disabling compactions & flushes 2024-11-14T12:54:34,709 INFO [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,,1731588868017.a572ae08c8738aa198c40f32aa4b5f96. 2024-11-14T12:54:34,709 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1731588868017.a572ae08c8738aa198c40f32aa4b5f96. 2024-11-14T12:54:34,709 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,,1731588868017.a572ae08c8738aa198c40f32aa4b5f96. after waiting 0 ms 2024-11-14T12:54:34,709 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,,1731588868017.a572ae08c8738aa198c40f32aa4b5f96. 2024-11-14T12:54:34,710 INFO [regionserver/669ef94e8b26:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T12:54:34,712 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/cadcf21958eaec53ee489ca0918e3ab7/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T12:54:34,713 INFO [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,xxx,1731588868017.cadcf21958eaec53ee489ca0918e3ab7. 2024-11-14T12:54:34,713 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for cadcf21958eaec53ee489ca0918e3ab7: Waiting for close lock at 1731588874705Running coprocessor pre-close hooks at 1731588874705Disabling compacts and flushes for region at 1731588874705Disabling writes for close at 1731588874705Writing region close event to WAL at 1731588874707 (+2 ms)Running coprocessor post-close hooks at 1731588874713 (+6 ms)Closed at 1731588874713 2024-11-14T12:54:34,714 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,xxx,1731588868017.cadcf21958eaec53ee489ca0918e3ab7. 2024-11-14T12:54:34,714 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/a572ae08c8738aa198c40f32aa4b5f96/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T12:54:34,714 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1358fa84af33d1799bd11ba7f4959a37, disabling compactions & flushes 2024-11-14T12:54:34,714 INFO [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,jjj,1731588868017.1358fa84af33d1799bd11ba7f4959a37. 2024-11-14T12:54:34,714 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1731588868017.1358fa84af33d1799bd11ba7f4959a37. 2024-11-14T12:54:34,714 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1731588868017.1358fa84af33d1799bd11ba7f4959a37. after waiting 0 ms 2024-11-14T12:54:34,714 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1731588868017.1358fa84af33d1799bd11ba7f4959a37. 2024-11-14T12:54:34,714 INFO [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,,1731588868017.a572ae08c8738aa198c40f32aa4b5f96. 2024-11-14T12:54:34,714 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a572ae08c8738aa198c40f32aa4b5f96: Waiting for close lock at 1731588874708Running coprocessor pre-close hooks at 1731588874708Disabling compacts and flushes for region at 1731588874708Disabling writes for close at 1731588874709 (+1 ms)Writing region close event to WAL at 1731588874709Running coprocessor post-close hooks at 1731588874714 (+5 ms)Closed at 1731588874714 2024-11-14T12:54:34,715 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,,1731588868017.a572ae08c8738aa198c40f32aa4b5f96. 2024-11-14T12:54:34,715 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 64b5bd4e6112417a55e871b36551c5b8, disabling compactions & flushes 2024-11-14T12:54:34,715 INFO [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,lll,1731588868017.64b5bd4e6112417a55e871b36551c5b8. 2024-11-14T12:54:34,715 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1731588868017.64b5bd4e6112417a55e871b36551c5b8. 2024-11-14T12:54:34,715 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,lll,1731588868017.64b5bd4e6112417a55e871b36551c5b8. after waiting 0 ms 2024-11-14T12:54:34,715 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,lll,1731588868017.64b5bd4e6112417a55e871b36551c5b8. 2024-11-14T12:54:34,717 DEBUG [RS_CLOSE_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/hbase/meta/1588230740/.tmp/info/5b42d50b9340419e898552dd7f336855 is 138, key is test,hhh,1731588868017.cdc4b47ed9fe07b56101f9e0e17abcd7./info:regioninfo/1731588873493/Put/seqid=0 2024-11-14T12:54:34,721 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/64b5bd4e6112417a55e871b36551c5b8/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T12:54:34,722 INFO [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,lll,1731588868017.64b5bd4e6112417a55e871b36551c5b8. 2024-11-14T12:54:34,722 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 64b5bd4e6112417a55e871b36551c5b8: Waiting for close lock at 1731588874715Running coprocessor pre-close hooks at 1731588874715Disabling compacts and flushes for region at 1731588874715Disabling writes for close at 1731588874715Writing region close event to WAL at 1731588874715Running coprocessor post-close hooks at 1731588874722 (+7 ms)Closed at 1731588874722 2024-11-14T12:54:34,722 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/1358fa84af33d1799bd11ba7f4959a37/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T12:54:34,722 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,lll,1731588868017.64b5bd4e6112417a55e871b36551c5b8. 2024-11-14T12:54:34,722 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b8044551b940cf51bfa7c7633c4df21d, disabling compactions & flushes 2024-11-14T12:54:34,722 INFO [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ddd,1731588868017.b8044551b940cf51bfa7c7633c4df21d. 2024-11-14T12:54:34,722 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1731588868017.b8044551b940cf51bfa7c7633c4df21d. 2024-11-14T12:54:34,722 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1731588868017.b8044551b940cf51bfa7c7633c4df21d. after waiting 0 ms 2024-11-14T12:54:34,723 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1731588868017.b8044551b940cf51bfa7c7633c4df21d. 2024-11-14T12:54:34,723 INFO [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,jjj,1731588868017.1358fa84af33d1799bd11ba7f4959a37. 2024-11-14T12:54:34,723 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1358fa84af33d1799bd11ba7f4959a37: Waiting for close lock at 1731588874714Running coprocessor pre-close hooks at 1731588874714Disabling compacts and flushes for region at 1731588874714Disabling writes for close at 1731588874714Writing region close event to WAL at 1731588874714Running coprocessor post-close hooks at 1731588874723 (+9 ms)Closed at 1731588874723 2024-11-14T12:54:34,723 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,jjj,1731588868017.1358fa84af33d1799bd11ba7f4959a37. 2024-11-14T12:54:34,723 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ea6e95809b33f853859fa227179bfe85, disabling compactions & flushes 2024-11-14T12:54:34,723 INFO [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ppp,1731588868017.ea6e95809b33f853859fa227179bfe85. 2024-11-14T12:54:34,723 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1731588868017.ea6e95809b33f853859fa227179bfe85. 2024-11-14T12:54:34,723 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1731588868017.ea6e95809b33f853859fa227179bfe85. after waiting 0 ms 2024-11-14T12:54:34,723 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1731588868017.ea6e95809b33f853859fa227179bfe85. 2024-11-14T12:54:34,732 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/b8044551b940cf51bfa7c7633c4df21d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T12:54:34,732 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/ea6e95809b33f853859fa227179bfe85/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T12:54:34,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741869_1045 (size=8174) 2024-11-14T12:54:34,733 INFO [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ddd,1731588868017.b8044551b940cf51bfa7c7633c4df21d. 2024-11-14T12:54:34,733 INFO [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ppp,1731588868017.ea6e95809b33f853859fa227179bfe85. 2024-11-14T12:54:34,733 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ea6e95809b33f853859fa227179bfe85: Waiting for close lock at 1731588874723Running coprocessor pre-close hooks at 1731588874723Disabling compacts and flushes for region at 1731588874723Disabling writes for close at 1731588874723Writing region close event to WAL at 1731588874724 (+1 ms)Running coprocessor post-close hooks at 1731588874733 (+9 ms)Closed at 1731588874733 2024-11-14T12:54:34,733 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b8044551b940cf51bfa7c7633c4df21d: Waiting for close lock at 1731588874722Running coprocessor pre-close hooks at 1731588874722Disabling compacts and flushes for region at 1731588874722Disabling writes for close at 1731588874722Writing region close event to WAL at 1731588874723 (+1 ms)Running coprocessor post-close hooks at 1731588874733 (+10 ms)Closed at 1731588874733 2024-11-14T12:54:34,733 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ppp,1731588868017.ea6e95809b33f853859fa227179bfe85. 2024-11-14T12:54:34,733 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ddd,1731588868017.b8044551b940cf51bfa7c7633c4df21d. 2024-11-14T12:54:34,734 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing bf47be17fd22bc204db28da07b30af2c, disabling compactions & flushes 2024-11-14T12:54:34,734 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f76ec25f51f69b1106342a7dfae65fb6, disabling compactions & flushes 2024-11-14T12:54:34,734 INFO [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,fff,1731588868017.bf47be17fd22bc204db28da07b30af2c. 2024-11-14T12:54:34,734 INFO [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ggg,1731588868017.f76ec25f51f69b1106342a7dfae65fb6. 2024-11-14T12:54:34,734 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1731588868017.f76ec25f51f69b1106342a7dfae65fb6. 2024-11-14T12:54:34,734 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1731588868017.bf47be17fd22bc204db28da07b30af2c. 2024-11-14T12:54:34,734 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1731588868017.f76ec25f51f69b1106342a7dfae65fb6. after waiting 0 ms 2024-11-14T12:54:34,734 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,fff,1731588868017.bf47be17fd22bc204db28da07b30af2c. after waiting 0 ms 2024-11-14T12:54:34,734 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1731588868017.f76ec25f51f69b1106342a7dfae65fb6. 2024-11-14T12:54:34,734 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,fff,1731588868017.bf47be17fd22bc204db28da07b30af2c. 2024-11-14T12:54:34,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43871 is added to blk_1073741869_1045 (size=8174) 2024-11-14T12:54:34,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36021 is added to blk_1073741869_1045 (size=8174) 2024-11-14T12:54:34,737 INFO [RS_CLOSE_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.64 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/hbase/meta/1588230740/.tmp/info/5b42d50b9340419e898552dd7f336855 2024-11-14T12:54:34,738 WARN [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,qqq,1731588868017.18ddd925c95b1b23296976da04f97e83., aborting... java.io.IOException: Failed to report opened region to master: test,qqq,1731588868017.18ddd925c95b1b23296976da04f97e83. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T12:54:34,738 ERROR [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2440): ***** ABORTING region server 669ef94e8b26,43475,1731588864980: Failed to open region test,qqq,1731588868017.18ddd925c95b1b23296976da04f97e83. and can not recover ***** java.io.IOException: Failed to report opened region to master: test,qqq,1731588868017.18ddd925c95b1b23296976da04f97e83. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T12:54:34,739 ERROR [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-14T12:54:34,739 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-14T12:54:34,739 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/bf47be17fd22bc204db28da07b30af2c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T12:54:34,740 INFO [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,fff,1731588868017.bf47be17fd22bc204db28da07b30af2c. 2024-11-14T12:54:34,740 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for bf47be17fd22bc204db28da07b30af2c: Waiting for close lock at 1731588874734Running coprocessor pre-close hooks at 1731588874734Disabling compacts and flushes for region at 1731588874734Disabling writes for close at 1731588874734Writing region close event to WAL at 1731588874734Running coprocessor post-close hooks at 1731588874740 (+6 ms)Closed at 1731588874740 2024-11-14T12:54:34,740 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,fff,1731588868017.bf47be17fd22bc204db28da07b30af2c. 2024-11-14T12:54:34,740 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 338b47e35e9a8b9dd3045d365aaf47bd, disabling compactions & flushes 2024-11-14T12:54:34,740 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-14T12:54:34,740 INFO [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ooo,1731588868017.338b47e35e9a8b9dd3045d365aaf47bd. 2024-11-14T12:54:34,740 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1731588868017.338b47e35e9a8b9dd3045d365aaf47bd. 2024-11-14T12:54:34,740 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1731588868017.338b47e35e9a8b9dd3045d365aaf47bd. after waiting 0 ms 2024-11-14T12:54:34,740 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1731588868017.338b47e35e9a8b9dd3045d365aaf47bd. 2024-11-14T12:54:34,741 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/f76ec25f51f69b1106342a7dfae65fb6/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T12:54:34,743 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-14T12:54:34,744 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-14T12:54:34,748 INFO [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ggg,1731588868017.f76ec25f51f69b1106342a7dfae65fb6. 2024-11-14T12:54:34,749 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f76ec25f51f69b1106342a7dfae65fb6: Waiting for close lock at 1731588874734Running coprocessor pre-close hooks at 1731588874734Disabling compacts and flushes for region at 1731588874734Disabling writes for close at 1731588874734Writing region close event to WAL at 1731588874735 (+1 ms)Running coprocessor post-close hooks at 1731588874748 (+13 ms)Closed at 1731588874748 2024-11-14T12:54:34,749 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ggg,1731588868017.f76ec25f51f69b1106342a7dfae65fb6. 2024-11-14T12:54:34,749 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 87eb074dbd46b3bc6e27fb0bf3291d5b, disabling compactions & flushes 2024-11-14T12:54:34,749 INFO [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,vvv,1731588868017.87eb074dbd46b3bc6e27fb0bf3291d5b. 2024-11-14T12:54:34,749 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1731588868017.87eb074dbd46b3bc6e27fb0bf3291d5b. 2024-11-14T12:54:34,749 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1731588868017.87eb074dbd46b3bc6e27fb0bf3291d5b. after waiting 0 ms 2024-11-14T12:54:34,749 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1731588868017.87eb074dbd46b3bc6e27fb0bf3291d5b. 2024-11-14T12:54:34,755 DEBUG [RS_CLOSE_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/hbase/meta/1588230740/.tmp/info/5b42d50b9340419e898552dd7f336855 as hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/hbase/meta/1588230740/info/5b42d50b9340419e898552dd7f336855 2024-11-14T12:54:34,756 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 512475328 }, "NonHeapMemoryUsage": { "committed": 163774464, "init": 7667712, "max": -1, "used": 161140280 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "669ef94e8b26", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 2, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 35786, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 104, "ProcessCallTime_min": 0, "ProcessCallTime_max": 30, "ProcessCallTime_mean": 2, "ProcessCallTime_25th_percentile": 7, "ProcessCallTime_median": 15, "ProcessCallTime_75th_percentile": 22, "ProcessCallTime_90th_percentile": 27, "ProcessCallTime_95th_percentile": 28, "ProcessCallTime_98th_percentile": 29, "ProcessCallTime_99th_percentile": 29, "ProcessCallTime_99.9th_percentile": 29, "ProcessCallTime_TimeRangeCount_0-1": 104, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 104, "QueueCallTime_min": 0, "QueueCallTime_max": 22, "QueueCallTime_mean": 1, "QueueCallTime_25th_percentile": 5, "QueueCallTime_median": 11, "QueueCallTime_75th_percentile": 16, "QueueCallTime_90th_percentile": 19, "QueueCallTime_95th_percentile": 20, "QueueCallTime_98th_percentile": 21, "QueueCallTime_99th_percentile": 21, "QueueCallTime_99.9th_percentile": 21, "QueueCallTime_TimeRangeCount_0-1": 104, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 104, "TotalCallTime_min": 0, "TotalCallTime_max": 30, "TotalCallTime_mean": 4, "TotalCallTime_25th_percentile": 7, "TotalCallTime_median": 15, "TotalCallTime_75th_percentile": 22, "TotalCallTime_90th_percentile": 27, "TotalCallTime_95th_percentile": 28, "TotalCallTime_98th_percentile": 29, "TotalCallTime_99th_percentile": 29, "TotalCallTime_99.9th_percentile": 29, "TotalCallTime_TimeRangeCount_0-1": 104, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 104, "ResponseSize_min": 0, "ResponseSize_max": 22359, "ResponseSize_mean": 657, "ResponseSize_25th_percentile": 5589, "ResponseSize_median": 11179, "ResponseSize_75th_percentile": 16769, "ResponseSize_90th_percentile": 20123, "ResponseSize_95th_percentile": 21241, "ResponseSize_98th_percentile": 21911, "ResponseSize_99th_percentile": 22135, "ResponseSize_99.9th_percentile": 22336, "ResponseSize_SizeRangeCount_0-10": 104, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 104, "RequestSize_min": 81, "RequestSize_max": 8095, "RequestSize_mean": 313, "RequestSize_25th_percentile": 2084, "RequestSize_median": 4088, "RequestSize_75th_percentile": 6091, "RequestSize_90th_percentile": 7293, "RequestSize_95th_percentile": 7694, "RequestSize_98th_percentile": 7934, "RequestSize_99th_percentile": 8014, "RequestSize_99.9th_percentile": 8086, "RequestSize_SizeRangeCount_0-10": 104, "sentBytes": 54772 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "669ef94e8b26", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:60896", "tag.serverName": "669ef94e8b26,43475,1731588864980", "tag.clusterId": "e7c3c559-5520-4497-bea0-7e8e506d06ae", "tag.Context": "regionserver", "tag.Hostname": "669ef94e8b26", "regionCount": 10, "storeCount": 9, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 2304, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1731588864980, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 3341, "localBytesRead": 3341, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 1.8, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 7474, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 9, "totalRowActionRequestCount": 9, "readRequestCount": 9, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 9, "rpcScanRequestCount": 9, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 9, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "ScanTime_TimeRangeCount_0-1": 9, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 46703, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 30111, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 1, "FlushOutputSize_min": 46703, "FlushOutputSize_max": 46703, "FlushOutputSize_mean": 46703, "FlushOutputSize_25th_percentile": 46703, "FlushOutputSize_median": 46703, "FlushOutputSize_75th_percentile": 46703, "FlushOutputSize_90th_percentile": 46703, "FlushOutputSize_95th_percentile": 46703, "FlushOutputSize_98th_percentile": 46703, "FlushOutputSize_99th_percentile": 46703, "FlushOutputSize_99.9th_percentile": 46703, "FlushOutputSize_SizeRangeCount_0-10": 1, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 1, "FlushMemstoreSize_min": 30111, "FlushMemstoreSize_max": 30111, "FlushMemstoreSize_mean": 30111, "FlushMemstoreSize_25th_percentile": 30111, "FlushMemstoreSize_median": 30111, "FlushMemstoreSize_75th_percentile": 30111, "FlushMemstoreSize_90th_percentile": 30111, "FlushMemstoreSize_95th_percentile": 30111, "FlushMemstoreSize_98th_percentile": 30111, "FlushMemstoreSize_99th_percentile": 30111, "FlushMemstoreSize_99.9th_percentile": 30111, "FlushMemstoreSize_SizeRangeCount_0-10": 1, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 1, "FlushTime_min": 662, "FlushTime_max": 662, "FlushTime_mean": 662, "FlushTime_25th_percentile": 662, "FlushTime_median": 662, "FlushTime_75th_percentile": 662, "FlushTime_90th_percentile": 662, "FlushTime_95th_percentile": 662, "FlushTime_98th_percentile": 662, "FlushTime_99th_percentile": 662, "FlushTime_99.9th_percentile": 662, "FlushTime_TimeRangeCount_0-1": 1, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 9, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "ScanSize_SizeRangeCount_0-10": 9, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-14T12:54:34,758 WARN [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2470): Unable to report fatal error to master org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=669ef94e8b26:45309 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:344) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportRSFatalError(RegionServerStatusProtos.java:17290) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.abort(HRegionServer.java:2467) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abortRegionServer(SingleProcessHBaseCluster.java:192) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$2.run(SingleProcessHBaseCluster.java:185) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abort(SingleProcessHBaseCluster.java:182) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.handleException(AssignRegionHandler.java:173) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:106) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=669ef94e8b26:45309 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 15 more Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 15 more 2024-11-14T12:54:34,759 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/338b47e35e9a8b9dd3045d365aaf47bd/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T12:54:34,760 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/87eb074dbd46b3bc6e27fb0bf3291d5b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T12:54:34,761 INFO [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ooo,1731588868017.338b47e35e9a8b9dd3045d365aaf47bd. 2024-11-14T12:54:34,761 INFO [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,vvv,1731588868017.87eb074dbd46b3bc6e27fb0bf3291d5b. 2024-11-14T12:54:34,761 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 338b47e35e9a8b9dd3045d365aaf47bd: Waiting for close lock at 1731588874740Running coprocessor pre-close hooks at 1731588874740Disabling compacts and flushes for region at 1731588874740Disabling writes for close at 1731588874740Writing region close event to WAL at 1731588874741 (+1 ms)Running coprocessor post-close hooks at 1731588874760 (+19 ms)Closed at 1731588874761 (+1 ms) 2024-11-14T12:54:34,761 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 87eb074dbd46b3bc6e27fb0bf3291d5b: Waiting for close lock at 1731588874749Running coprocessor pre-close hooks at 1731588874749Disabling compacts and flushes for region at 1731588874749Disabling writes for close at 1731588874749Writing region close event to WAL at 1731588874750 (+1 ms)Running coprocessor post-close hooks at 1731588874760 (+10 ms)Closed at 1731588874761 (+1 ms) 2024-11-14T12:54:34,761 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ooo,1731588868017.338b47e35e9a8b9dd3045d365aaf47bd. 2024-11-14T12:54:34,761 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,vvv,1731588868017.87eb074dbd46b3bc6e27fb0bf3291d5b. 2024-11-14T12:54:34,761 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 03f289a2805f0b2f037581e9390ba191, disabling compactions & flushes 2024-11-14T12:54:34,761 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 23f7fbc112f1353900c1181e3ed36707, disabling compactions & flushes 2024-11-14T12:54:34,761 INFO [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,iii,1731588868017.03f289a2805f0b2f037581e9390ba191. 2024-11-14T12:54:34,761 INFO [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,eee,1731588868017.23f7fbc112f1353900c1181e3ed36707. 2024-11-14T12:54:34,761 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1731588868017.03f289a2805f0b2f037581e9390ba191. 2024-11-14T12:54:34,761 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1731588868017.23f7fbc112f1353900c1181e3ed36707. 2024-11-14T12:54:34,761 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,iii,1731588868017.03f289a2805f0b2f037581e9390ba191. after waiting 0 ms 2024-11-14T12:54:34,761 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,eee,1731588868017.23f7fbc112f1353900c1181e3ed36707. after waiting 0 ms 2024-11-14T12:54:34,761 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,iii,1731588868017.03f289a2805f0b2f037581e9390ba191. 2024-11-14T12:54:34,761 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,eee,1731588868017.23f7fbc112f1353900c1181e3ed36707. 2024-11-14T12:54:34,765 INFO [RS_CLOSE_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/hbase/meta/1588230740/info/5b42d50b9340419e898552dd7f336855, entries=24, sequenceid=75, filesize=8.0 K 2024-11-14T12:54:34,767 INFO [RS_CLOSE_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.64 KB/2705, heapSize ~4.66 KB/4768, currentSize=0 B/0 for 1588230740 in 66ms, sequenceid=75, compaction requested=false 2024-11-14T12:54:34,768 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/03f289a2805f0b2f037581e9390ba191/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T12:54:34,768 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/23f7fbc112f1353900c1181e3ed36707/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-14T12:54:34,768 INFO [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,iii,1731588868017.03f289a2805f0b2f037581e9390ba191. 2024-11-14T12:54:34,769 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 03f289a2805f0b2f037581e9390ba191: Waiting for close lock at 1731588874761Running coprocessor pre-close hooks at 1731588874761Disabling compacts and flushes for region at 1731588874761Disabling writes for close at 1731588874761Writing region close event to WAL at 1731588874762 (+1 ms)Running coprocessor post-close hooks at 1731588874768 (+6 ms)Closed at 1731588874768 2024-11-14T12:54:34,769 INFO [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,eee,1731588868017.23f7fbc112f1353900c1181e3ed36707. 2024-11-14T12:54:34,769 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,iii,1731588868017.03f289a2805f0b2f037581e9390ba191. 2024-11-14T12:54:34,769 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 23f7fbc112f1353900c1181e3ed36707: Waiting for close lock at 1731588874761Running coprocessor pre-close hooks at 1731588874761Disabling compacts and flushes for region at 1731588874761Disabling writes for close at 1731588874761Writing region close event to WAL at 1731588874762 (+1 ms)Running coprocessor post-close hooks at 1731588874769 (+7 ms)Closed at 1731588874769 2024-11-14T12:54:34,769 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 22b9f10374fc1ea7a876d476ce0b083d, disabling compactions & flushes 2024-11-14T12:54:34,769 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,eee,1731588868017.23f7fbc112f1353900c1181e3ed36707. 2024-11-14T12:54:34,769 INFO [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,www,1731588868017.22b9f10374fc1ea7a876d476ce0b083d. 2024-11-14T12:54:34,769 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1731588868017.22b9f10374fc1ea7a876d476ce0b083d. 2024-11-14T12:54:34,769 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,www,1731588868017.22b9f10374fc1ea7a876d476ce0b083d. after waiting 0 ms 2024-11-14T12:54:34,769 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,www,1731588868017.22b9f10374fc1ea7a876d476ce0b083d. 2024-11-14T12:54:34,769 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1d33b9e9c25fcfc654d257646677381c, disabling compactions & flushes 2024-11-14T12:54:34,769 INFO [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,sss,1731588868017.1d33b9e9c25fcfc654d257646677381c. 2024-11-14T12:54:34,769 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1731588868017.1d33b9e9c25fcfc654d257646677381c. 2024-11-14T12:54:34,769 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,sss,1731588868017.1d33b9e9c25fcfc654d257646677381c. after waiting 0 ms 2024-11-14T12:54:34,769 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,sss,1731588868017.1d33b9e9c25fcfc654d257646677381c. 2024-11-14T12:54:34,774 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/22b9f10374fc1ea7a876d476ce0b083d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T12:54:34,774 INFO [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,www,1731588868017.22b9f10374fc1ea7a876d476ce0b083d. 2024-11-14T12:54:34,774 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 22b9f10374fc1ea7a876d476ce0b083d: Waiting for close lock at 1731588874769Running coprocessor pre-close hooks at 1731588874769Disabling compacts and flushes for region at 1731588874769Disabling writes for close at 1731588874769Writing region close event to WAL at 1731588874770 (+1 ms)Running coprocessor post-close hooks at 1731588874774 (+4 ms)Closed at 1731588874774 2024-11-14T12:54:34,775 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,www,1731588868017.22b9f10374fc1ea7a876d476ce0b083d. 2024-11-14T12:54:34,775 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f8c0d62856832eec9f9ca20bb6db602c, disabling compactions & flushes 2024-11-14T12:54:34,775 INFO [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,uuu,1731588868017.f8c0d62856832eec9f9ca20bb6db602c. 2024-11-14T12:54:34,775 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1731588868017.f8c0d62856832eec9f9ca20bb6db602c. 2024-11-14T12:54:34,775 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1731588868017.f8c0d62856832eec9f9ca20bb6db602c. after waiting 0 ms 2024-11-14T12:54:34,775 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1731588868017.f8c0d62856832eec9f9ca20bb6db602c. 2024-11-14T12:54:34,776 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/1d33b9e9c25fcfc654d257646677381c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T12:54:34,776 INFO [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,sss,1731588868017.1d33b9e9c25fcfc654d257646677381c. 2024-11-14T12:54:34,777 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1d33b9e9c25fcfc654d257646677381c: Waiting for close lock at 1731588874769Running coprocessor pre-close hooks at 1731588874769Disabling compacts and flushes for region at 1731588874769Disabling writes for close at 1731588874769Writing region close event to WAL at 1731588874770 (+1 ms)Running coprocessor post-close hooks at 1731588874776 (+6 ms)Closed at 1731588874776 2024-11-14T12:54:34,777 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,sss,1731588868017.1d33b9e9c25fcfc654d257646677381c. 2024-11-14T12:54:34,777 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e90cc230c4f13fece446e3e8420dddbe, disabling compactions & flushes 2024-11-14T12:54:34,777 INFO [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,mmm,1731588868017.e90cc230c4f13fece446e3e8420dddbe. 2024-11-14T12:54:34,777 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1731588868017.e90cc230c4f13fece446e3e8420dddbe. 2024-11-14T12:54:34,777 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1731588868017.e90cc230c4f13fece446e3e8420dddbe. after waiting 0 ms 2024-11-14T12:54:34,777 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1731588868017.e90cc230c4f13fece446e3e8420dddbe. 2024-11-14T12:54:34,780 DEBUG [RS_CLOSE_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/hbase/meta/1588230740/recovered.edits/78.seqid, newMaxSeqId=78, maxSeqId=64 2024-11-14T12:54:34,781 DEBUG [RS_CLOSE_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T12:54:34,781 INFO [RS_CLOSE_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T12:54:34,781 DEBUG [RS_CLOSE_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731588874701Running coprocessor pre-close hooks at 1731588874701Disabling compacts and flushes for region at 1731588874701Disabling writes for close at 1731588874701Obtaining lock to block concurrent updates at 1731588874701Preparing flush snapshotting stores in 1588230740 at 1731588874701Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=2705, getHeapSize=5488, getOffHeapSize=0, getCellsCount=24 at 1731588874710 (+9 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731588874711 (+1 ms)Flushing 1588230740/info: creating writer at 1731588874711Flushing 1588230740/info: appending metadata at 1731588874717 (+6 ms)Flushing 1588230740/info: closing flushed file at 1731588874717Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6b16112c: reopening flushed file at 1731588874754 (+37 ms)Finished flush of dataSize ~2.64 KB/2705, heapSize ~4.66 KB/4768, currentSize=0 B/0 for 1588230740 in 66ms, sequenceid=75, compaction requested=false at 1731588874767 (+13 ms)Writing region close event to WAL at 1731588874770 (+3 ms)Running coprocessor post-close hooks at 1731588874780 (+10 ms)Closed at 1731588874781 (+1 ms) 2024-11-14T12:54:34,781 DEBUG [RS_CLOSE_META-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-14T12:54:34,784 INFO [regionserver/669ef94e8b26:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T12:54:34,784 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/f8c0d62856832eec9f9ca20bb6db602c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T12:54:34,784 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/e90cc230c4f13fece446e3e8420dddbe/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T12:54:34,785 INFO [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,uuu,1731588868017.f8c0d62856832eec9f9ca20bb6db602c. 2024-11-14T12:54:34,785 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f8c0d62856832eec9f9ca20bb6db602c: Waiting for close lock at 1731588874775Running coprocessor pre-close hooks at 1731588874775Disabling compacts and flushes for region at 1731588874775Disabling writes for close at 1731588874775Writing region close event to WAL at 1731588874775Running coprocessor post-close hooks at 1731588874785 (+10 ms)Closed at 1731588874785 2024-11-14T12:54:34,785 INFO [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,mmm,1731588868017.e90cc230c4f13fece446e3e8420dddbe. 2024-11-14T12:54:34,785 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e90cc230c4f13fece446e3e8420dddbe: Waiting for close lock at 1731588874777Running coprocessor pre-close hooks at 1731588874777Disabling compacts and flushes for region at 1731588874777Disabling writes for close at 1731588874777Writing region close event to WAL at 1731588874779 (+2 ms)Running coprocessor post-close hooks at 1731588874785 (+6 ms)Closed at 1731588874785 2024-11-14T12:54:34,785 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,uuu,1731588868017.f8c0d62856832eec9f9ca20bb6db602c. 2024-11-14T12:54:34,785 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,mmm,1731588868017.e90cc230c4f13fece446e3e8420dddbe. 2024-11-14T12:54:34,785 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 333a3eba2bb2d318c041beb4642f5ef3, disabling compactions & flushes 2024-11-14T12:54:34,785 INFO [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,kkk,1731588868017.333a3eba2bb2d318c041beb4642f5ef3. 2024-11-14T12:54:34,785 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1731588868017.333a3eba2bb2d318c041beb4642f5ef3. 2024-11-14T12:54:34,785 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1731588868017.333a3eba2bb2d318c041beb4642f5ef3. after waiting 0 ms 2024-11-14T12:54:34,785 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1731588868017.333a3eba2bb2d318c041beb4642f5ef3. 2024-11-14T12:54:34,789 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/333a3eba2bb2d318c041beb4642f5ef3/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T12:54:34,790 INFO [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,kkk,1731588868017.333a3eba2bb2d318c041beb4642f5ef3. 2024-11-14T12:54:34,790 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 333a3eba2bb2d318c041beb4642f5ef3: Waiting for close lock at 1731588874785Running coprocessor pre-close hooks at 1731588874785Disabling compacts and flushes for region at 1731588874785Disabling writes for close at 1731588874785Writing region close event to WAL at 1731588874785Running coprocessor post-close hooks at 1731588874790 (+5 ms)Closed at 1731588874790 2024-11-14T12:54:34,790 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,kkk,1731588868017.333a3eba2bb2d318c041beb4642f5ef3. 2024-11-14T12:54:34,790 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a5d884ba0737bc572475e022a6ed1bc8, disabling compactions & flushes 2024-11-14T12:54:34,790 INFO [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,rrr,1731588868017.a5d884ba0737bc572475e022a6ed1bc8. 2024-11-14T12:54:34,790 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1731588868017.a5d884ba0737bc572475e022a6ed1bc8. 2024-11-14T12:54:34,790 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1731588868017.a5d884ba0737bc572475e022a6ed1bc8. after waiting 0 ms 2024-11-14T12:54:34,790 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1731588868017.a5d884ba0737bc572475e022a6ed1bc8. 2024-11-14T12:54:34,792 WARN [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,zzz,1731588868017.2201f29440b3aa3259741bbb374e209d., aborting... java.io.IOException: Failed to report opened region to master: test,zzz,1731588868017.2201f29440b3aa3259741bbb374e209d. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T12:54:34,792 ERROR [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2440): ***** ABORTING region server 669ef94e8b26,46877,1731588865107: Failed to open region test,zzz,1731588868017.2201f29440b3aa3259741bbb374e209d. and can not recover ***** java.io.IOException: Failed to report opened region to master: test,zzz,1731588868017.2201f29440b3aa3259741bbb374e209d. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T12:54:34,792 ERROR [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-14T12:54:34,792 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-14T12:54:34,793 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-14T12:54:34,794 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-14T12:54:34,794 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-14T12:54:34,795 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/a5d884ba0737bc572475e022a6ed1bc8/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T12:54:34,796 INFO [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,rrr,1731588868017.a5d884ba0737bc572475e022a6ed1bc8. 2024-11-14T12:54:34,796 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a5d884ba0737bc572475e022a6ed1bc8: Waiting for close lock at 1731588874790Running coprocessor pre-close hooks at 1731588874790Disabling compacts and flushes for region at 1731588874790Disabling writes for close at 1731588874790Writing region close event to WAL at 1731588874791 (+1 ms)Running coprocessor post-close hooks at 1731588874796 (+5 ms)Closed at 1731588874796 2024-11-14T12:54:34,796 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,rrr,1731588868017.a5d884ba0737bc572475e022a6ed1bc8. 2024-11-14T12:54:34,796 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c2ccfc8c17ab1571bc4ea6d526017f9f, disabling compactions & flushes 2024-11-14T12:54:34,796 INFO [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,bbb,1731588868017.c2ccfc8c17ab1571bc4ea6d526017f9f. 2024-11-14T12:54:34,796 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1731588868017.c2ccfc8c17ab1571bc4ea6d526017f9f. 2024-11-14T12:54:34,796 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1731588868017.c2ccfc8c17ab1571bc4ea6d526017f9f. after waiting 0 ms 2024-11-14T12:54:34,796 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1731588868017.c2ccfc8c17ab1571bc4ea6d526017f9f. 2024-11-14T12:54:34,798 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 518766784 }, "NonHeapMemoryUsage": { "committed": 164102144, "init": 7667712, "max": -1, "used": 161493032 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "669ef94e8b26", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 2, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 35786, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 104, "ProcessCallTime_min": 0, "ProcessCallTime_max": 30, "ProcessCallTime_mean": 2, "ProcessCallTime_25th_percentile": 7, "ProcessCallTime_median": 15, "ProcessCallTime_75th_percentile": 22, "ProcessCallTime_90th_percentile": 27, "ProcessCallTime_95th_percentile": 28, "ProcessCallTime_98th_percentile": 29, "ProcessCallTime_99th_percentile": 29, "ProcessCallTime_99.9th_percentile": 29, "ProcessCallTime_TimeRangeCount_0-1": 104, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 104, "QueueCallTime_min": 0, "QueueCallTime_max": 22, "QueueCallTime_mean": 1, "QueueCallTime_25th_percentile": 5, "QueueCallTime_median": 11, "QueueCallTime_75th_percentile": 16, "QueueCallTime_90th_percentile": 19, "QueueCallTime_95th_percentile": 20, "QueueCallTime_98th_percentile": 21, "QueueCallTime_99th_percentile": 21, "QueueCallTime_99.9th_percentile": 21, "QueueCallTime_TimeRangeCount_0-1": 104, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 104, "TotalCallTime_min": 0, "TotalCallTime_max": 30, "TotalCallTime_mean": 4, "TotalCallTime_25th_percentile": 7, "TotalCallTime_median": 15, "TotalCallTime_75th_percentile": 22, "TotalCallTime_90th_percentile": 27, "TotalCallTime_95th_percentile": 28, "TotalCallTime_98th_percentile": 29, "TotalCallTime_99th_percentile": 29, "TotalCallTime_99.9th_percentile": 29, "TotalCallTime_TimeRangeCount_0-1": 104, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 104, "ResponseSize_min": 0, "ResponseSize_max": 22359, "ResponseSize_mean": 657, "ResponseSize_25th_percentile": 5589, "ResponseSize_median": 11179, "ResponseSize_75th_percentile": 16769, "ResponseSize_90th_percentile": 20123, "ResponseSize_95th_percentile": 21241, "ResponseSize_98th_percentile": 21911, "ResponseSize_99th_percentile": 22135, "ResponseSize_99.9th_percentile": 22336, "ResponseSize_SizeRangeCount_0-10": 104, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 104, "RequestSize_min": 81, "RequestSize_max": 8095, "RequestSize_mean": 313, "RequestSize_25th_percentile": 2084, "RequestSize_median": 4088, "RequestSize_75th_percentile": 6091, "RequestSize_90th_percentile": 7293, "RequestSize_95th_percentile": 7694, "RequestSize_98th_percentile": 7934, "RequestSize_99th_percentile": 8014, "RequestSize_99.9th_percentile": 8086, "RequestSize_SizeRangeCount_0-10": 104, "sentBytes": 54772 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "669ef94e8b26", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:60896", "tag.serverName": "669ef94e8b26,43475,1731588864980", "tag.clusterId": "e7c3c559-5520-4497-bea0-7e8e506d06ae", "tag.Context": "regionserver", "tag.Hostname": "669ef94e8b26", "regionCount": 10, "storeCount": 9, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 2304, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1731588864980, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 3341, "localBytesRead": 3341, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 1.8, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 7474, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 9, "totalRowActionRequestCount": 9, "readRequestCount": 9, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 9, "rpcScanRequestCount": 9, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 9, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "ScanTime_TimeRangeCount_0-1": 9, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 46703, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 30111, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 1, "FlushOutputSize_min": 46703, "FlushOutputSize_max": 46703, "FlushOutputSize_mean": 46703, "FlushOutputSize_25th_percentile": 46703, "FlushOutputSize_median": 46703, "FlushOutputSize_75th_percentile": 46703, "FlushOutputSize_90th_percentile": 46703, "FlushOutputSize_95th_percentile": 46703, "FlushOutputSize_98th_percentile": 46703, "FlushOutputSize_99th_percentile": 46703, "FlushOutputSize_99.9th_percentile": 46703, "FlushOutputSize_SizeRangeCount_0-10": 1, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 1, "FlushMemstoreSize_min": 30111, "FlushMemstoreSize_max": 30111, "FlushMemstoreSize_mean": 30111, "FlushMemstoreSize_25th_percentile": 30111, "FlushMemstoreSize_median": 30111, "FlushMemstoreSize_75th_percentile": 30111, "FlushMemstoreSize_90th_percentile": 30111, "FlushMemstoreSize_95th_percentile": 30111, "FlushMemstoreSize_98th_percentile": 30111, "FlushMemstoreSize_99th_percentile": 30111, "FlushMemstoreSize_99.9th_percentile": 30111, "FlushMemstoreSize_SizeRangeCount_0-10": 1, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 1, "FlushTime_min": 662, "FlushTime_max": 662, "FlushTime_mean": 662, "FlushTime_25th_percentile": 662, "FlushTime_median": 662, "FlushTime_75th_percentile": 662, "FlushTime_90th_percentile": 662, "FlushTime_95th_percentile": 662, "FlushTime_98th_percentile": 662, "FlushTime_99th_percentile": 662, "FlushTime_99.9th_percentile": 662, "FlushTime_TimeRangeCount_0-1": 1, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 9, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "ScanSize_SizeRangeCount_0-10": 9, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-14T12:54:34,799 WARN [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2470): Unable to report fatal error to master org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=669ef94e8b26:45309 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:344) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportRSFatalError(RegionServerStatusProtos.java:17290) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.abort(HRegionServer.java:2467) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abortRegionServer(SingleProcessHBaseCluster.java:192) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$2.run(SingleProcessHBaseCluster.java:185) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abort(SingleProcessHBaseCluster.java:182) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.handleException(AssignRegionHandler.java:173) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:106) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=669ef94e8b26:45309 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 15 more Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 15 more 2024-11-14T12:54:34,799 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(132): Open test,nnn,1731588868017.4f16209fa220cf1b7a18b891e4fe53c6. 2024-11-14T12:54:34,799 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7752): Opening region: {ENCODED => 4f16209fa220cf1b7a18b891e4fe53c6, NAME => 'test,nnn,1731588868017.4f16209fa220cf1b7a18b891e4fe53c6.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-14T12:54:34,800 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4f16209fa220cf1b7a18b891e4fe53c6 2024-11-14T12:54:34,800 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(898): Instantiated test,nnn,1731588868017.4f16209fa220cf1b7a18b891e4fe53c6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:34,800 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7794): checking encryption for 4f16209fa220cf1b7a18b891e4fe53c6 2024-11-14T12:54:34,800 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7797): checking classloading for 4f16209fa220cf1b7a18b891e4fe53c6 2024-11-14T12:54:34,801 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/c2ccfc8c17ab1571bc4ea6d526017f9f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T12:54:34,801 INFO [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,bbb,1731588868017.c2ccfc8c17ab1571bc4ea6d526017f9f. 2024-11-14T12:54:34,802 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c2ccfc8c17ab1571bc4ea6d526017f9f: Waiting for close lock at 1731588874796Running coprocessor pre-close hooks at 1731588874796Disabling compacts and flushes for region at 1731588874796Disabling writes for close at 1731588874796Writing region close event to WAL at 1731588874797 (+1 ms)Running coprocessor post-close hooks at 1731588874801 (+4 ms)Closed at 1731588874801 2024-11-14T12:54:34,802 INFO [StoreOpener-4f16209fa220cf1b7a18b891e4fe53c6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4f16209fa220cf1b7a18b891e4fe53c6 2024-11-14T12:54:34,802 DEBUG [RS_CLOSE_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,bbb,1731588868017.c2ccfc8c17ab1571bc4ea6d526017f9f. 2024-11-14T12:54:34,803 INFO [StoreOpener-4f16209fa220cf1b7a18b891e4fe53c6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4f16209fa220cf1b7a18b891e4fe53c6 columnFamilyName family 2024-11-14T12:54:34,803 DEBUG [StoreOpener-4f16209fa220cf1b7a18b891e4fe53c6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T12:54:34,803 INFO [StoreOpener-4f16209fa220cf1b7a18b891e4fe53c6-1 {}] regionserver.HStore(327): Store=4f16209fa220cf1b7a18b891e4fe53c6/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T12:54:34,803 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1038): replaying wal for 4f16209fa220cf1b7a18b891e4fe53c6 2024-11-14T12:54:34,804 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/4f16209fa220cf1b7a18b891e4fe53c6 2024-11-14T12:54:34,806 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/4f16209fa220cf1b7a18b891e4fe53c6 2024-11-14T12:54:34,806 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1048): stopping wal replay for 4f16209fa220cf1b7a18b891e4fe53c6 2024-11-14T12:54:34,806 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1060): Cleaning up temporary data for 4f16209fa220cf1b7a18b891e4fe53c6 2024-11-14T12:54:34,808 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1093): writing seq id for 4f16209fa220cf1b7a18b891e4fe53c6 2024-11-14T12:54:34,809 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1114): Opened 4f16209fa220cf1b7a18b891e4fe53c6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62537847, jitterRate=-0.06811346113681793}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T12:54:34,809 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4f16209fa220cf1b7a18b891e4fe53c6 2024-11-14T12:54:34,809 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1006): Region open journal for 4f16209fa220cf1b7a18b891e4fe53c6: Running coprocessor pre-open hook at 1731588874800Writing region info on filesystem at 1731588874800Initializing all the Stores at 1731588874801 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731588874801Cleaning up temporary data from old regions at 1731588874806 (+5 ms)Running coprocessor post-open hooks at 1731588874809 (+3 ms)Region opened successfully at 1731588874809 2024-11-14T12:54:34,810 WARN [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,nnn,1731588868017.4f16209fa220cf1b7a18b891e4fe53c6., aborting... org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 669ef94e8b26,46877,1731588865107 aborting at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkOpen(RSRpcServices.java:1463) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2235) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T12:54:34,810 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2435): Abort already in progress. Ignoring the current request with reason: Failed to open region test,nnn,1731588868017.4f16209fa220cf1b7a18b891e4fe53c6. and can not recover 2024-11-14T12:54:34,810 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(132): Open test,hhh,1731588868017.cdc4b47ed9fe07b56101f9e0e17abcd7. 2024-11-14T12:54:34,811 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7752): Opening region: {ENCODED => cdc4b47ed9fe07b56101f9e0e17abcd7, NAME => 'test,hhh,1731588868017.cdc4b47ed9fe07b56101f9e0e17abcd7.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-14T12:54:34,811 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test cdc4b47ed9fe07b56101f9e0e17abcd7 2024-11-14T12:54:34,811 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(898): Instantiated test,hhh,1731588868017.cdc4b47ed9fe07b56101f9e0e17abcd7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:34,811 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7794): checking encryption for cdc4b47ed9fe07b56101f9e0e17abcd7 2024-11-14T12:54:34,811 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7797): checking classloading for cdc4b47ed9fe07b56101f9e0e17abcd7 2024-11-14T12:54:34,813 INFO [StoreOpener-cdc4b47ed9fe07b56101f9e0e17abcd7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region cdc4b47ed9fe07b56101f9e0e17abcd7 2024-11-14T12:54:34,813 INFO [StoreOpener-cdc4b47ed9fe07b56101f9e0e17abcd7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cdc4b47ed9fe07b56101f9e0e17abcd7 columnFamilyName family 2024-11-14T12:54:34,813 DEBUG [StoreOpener-cdc4b47ed9fe07b56101f9e0e17abcd7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T12:54:34,814 INFO [StoreOpener-cdc4b47ed9fe07b56101f9e0e17abcd7-1 {}] regionserver.HStore(327): Store=cdc4b47ed9fe07b56101f9e0e17abcd7/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T12:54:34,814 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1038): replaying wal for cdc4b47ed9fe07b56101f9e0e17abcd7 2024-11-14T12:54:34,815 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/cdc4b47ed9fe07b56101f9e0e17abcd7 2024-11-14T12:54:34,816 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/cdc4b47ed9fe07b56101f9e0e17abcd7 2024-11-14T12:54:34,816 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1048): stopping wal replay for cdc4b47ed9fe07b56101f9e0e17abcd7 2024-11-14T12:54:34,817 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1060): Cleaning up temporary data for cdc4b47ed9fe07b56101f9e0e17abcd7 2024-11-14T12:54:34,818 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1093): writing seq id for cdc4b47ed9fe07b56101f9e0e17abcd7 2024-11-14T12:54:34,819 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1114): Opened cdc4b47ed9fe07b56101f9e0e17abcd7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65774407, jitterRate=-0.019884958863258362}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T12:54:34,819 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cdc4b47ed9fe07b56101f9e0e17abcd7 2024-11-14T12:54:34,819 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1006): Region open journal for cdc4b47ed9fe07b56101f9e0e17abcd7: Running coprocessor pre-open hook at 1731588874811Writing region info on filesystem at 1731588874811Initializing all the Stores at 1731588874812 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731588874812Cleaning up temporary data from old regions at 1731588874817 (+5 ms)Running coprocessor post-open hooks at 1731588874819 (+2 ms)Region opened successfully at 1731588874819 2024-11-14T12:54:34,820 WARN [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,hhh,1731588868017.cdc4b47ed9fe07b56101f9e0e17abcd7., aborting... org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 669ef94e8b26,46877,1731588865107 aborting at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkOpen(RSRpcServices.java:1463) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2235) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T12:54:34,821 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2435): Abort already in progress. Ignoring the current request with reason: Failed to open region test,hhh,1731588868017.cdc4b47ed9fe07b56101f9e0e17abcd7. and can not recover 2024-11-14T12:54:34,821 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] handler.AssignRegionHandler(132): Open test,ttt,1731588868017.c8ddc4f331afb3a3007ef5c49cdf6ab8. 2024-11-14T12:54:34,821 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7752): Opening region: {ENCODED => c8ddc4f331afb3a3007ef5c49cdf6ab8, NAME => 'test,ttt,1731588868017.c8ddc4f331afb3a3007ef5c49cdf6ab8.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-14T12:54:34,821 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c8ddc4f331afb3a3007ef5c49cdf6ab8 2024-11-14T12:54:34,821 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(898): Instantiated test,ttt,1731588868017.c8ddc4f331afb3a3007ef5c49cdf6ab8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T12:54:34,821 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7794): checking encryption for c8ddc4f331afb3a3007ef5c49cdf6ab8 2024-11-14T12:54:34,821 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7797): checking classloading for c8ddc4f331afb3a3007ef5c49cdf6ab8 2024-11-14T12:54:34,823 INFO [StoreOpener-c8ddc4f331afb3a3007ef5c49cdf6ab8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c8ddc4f331afb3a3007ef5c49cdf6ab8 2024-11-14T12:54:34,824 INFO [StoreOpener-c8ddc4f331afb3a3007ef5c49cdf6ab8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c8ddc4f331afb3a3007ef5c49cdf6ab8 columnFamilyName family 2024-11-14T12:54:34,824 DEBUG [StoreOpener-c8ddc4f331afb3a3007ef5c49cdf6ab8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T12:54:34,825 INFO [StoreOpener-c8ddc4f331afb3a3007ef5c49cdf6ab8-1 {}] regionserver.HStore(327): Store=c8ddc4f331afb3a3007ef5c49cdf6ab8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T12:54:34,825 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1038): replaying wal for c8ddc4f331afb3a3007ef5c49cdf6ab8 2024-11-14T12:54:34,826 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/c8ddc4f331afb3a3007ef5c49cdf6ab8 2024-11-14T12:54:34,828 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/data/default/test/c8ddc4f331afb3a3007ef5c49cdf6ab8 2024-11-14T12:54:34,828 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1048): stopping wal replay for c8ddc4f331afb3a3007ef5c49cdf6ab8 2024-11-14T12:54:34,828 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1060): Cleaning up temporary data for c8ddc4f331afb3a3007ef5c49cdf6ab8 2024-11-14T12:54:34,830 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1093): writing seq id for c8ddc4f331afb3a3007ef5c49cdf6ab8 2024-11-14T12:54:34,831 INFO [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1114): Opened c8ddc4f331afb3a3007ef5c49cdf6ab8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67229510, jitterRate=0.0017977654933929443}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T12:54:34,831 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c8ddc4f331afb3a3007ef5c49cdf6ab8 2024-11-14T12:54:34,831 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1006): Region open journal for c8ddc4f331afb3a3007ef5c49cdf6ab8: Running coprocessor pre-open hook at 1731588874822Writing region info on filesystem at 1731588874822Initializing all the Stores at 1731588874823 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731588874823Cleaning up temporary data from old regions at 1731588874828 (+5 ms)Running coprocessor post-open hooks at 1731588874831 (+3 ms)Region opened successfully at 1731588874831 2024-11-14T12:54:34,832 WARN [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,ttt,1731588868017.c8ddc4f331afb3a3007ef5c49cdf6ab8., aborting... org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 669ef94e8b26,46877,1731588865107 aborting at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkOpen(RSRpcServices.java:1463) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2235) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T12:54:34,832 DEBUG [RS_OPEN_REGION-regionserver/669ef94e8b26:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegionServer(2435): Abort already in progress. Ignoring the current request with reason: Failed to open region test,ttt,1731588868017.c8ddc4f331afb3a3007ef5c49cdf6ab8. and can not recover 2024-11-14T12:54:34,889 INFO [regionserver/669ef94e8b26:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-14T12:54:34,889 INFO [regionserver/669ef94e8b26:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-14T12:54:34,901 INFO [RS:0;669ef94e8b26:43475 {}] regionserver.HRegionServer(976): stopping server 669ef94e8b26,43475,1731588864980; all regions closed. 2024-11-14T12:54:34,901 INFO [RS:1;669ef94e8b26:46877 {}] regionserver.HRegionServer(976): stopping server 669ef94e8b26,46877,1731588865107; all regions closed. 2024-11-14T12:54:34,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741835_1011 (size=5998) 2024-11-14T12:54:34,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36021 is added to blk_1073741835_1011 (size=5998) 2024-11-14T12:54:34,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43871 is added to blk_1073741865_1041 (size=4189) 2024-11-14T12:54:34,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741865_1041 (size=4189) 2024-11-14T12:54:34,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36021 is added to blk_1073741865_1041 (size=4189) 2024-11-14T12:54:34,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43871 is added to blk_1073741835_1011 (size=5998) 2024-11-14T12:54:34,906 DEBUG [RS:1;669ef94e8b26:46877 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T12:54:34,906 INFO [RS:1;669ef94e8b26:46877 {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T12:54:34,906 INFO [RS:1;669ef94e8b26:46877 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T12:54:34,906 INFO [RS:1;669ef94e8b26:46877 {}] hbase.ChoreService(370): Chore service for: regionserver/669ef94e8b26:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-14T12:54:34,906 INFO [RS:1;669ef94e8b26:46877 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-14T12:54:34,906 INFO [regionserver/669ef94e8b26:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T12:54:34,906 INFO [RS:1;669ef94e8b26:46877 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-14T12:54:34,906 INFO [RS:1;669ef94e8b26:46877 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-14T12:54:34,906 INFO [RS:1;669ef94e8b26:46877 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T12:54:34,907 INFO [RS:1;669ef94e8b26:46877 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:46877 2024-11-14T12:54:34,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43871 is added to blk_1073741834_1010 (size=6795) 2024-11-14T12:54:34,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741834_1010 (size=6795) 2024-11-14T12:54:34,912 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/WALs/669ef94e8b26,43475,1731588864980/669ef94e8b26%2C43475%2C1731588864980.1731588867077 not finished, retry = 0 2024-11-14T12:54:34,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36021 is added to blk_1073741834_1010 (size=6795) 2024-11-14T12:54:34,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46877-0x1013ab2e72b0002, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/669ef94e8b26,46877,1731588865107 2024-11-14T12:54:34,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45309-0x1013ab2e72b0000, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T12:54:34,922 INFO [RS:1;669ef94e8b26:46877 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T12:54:34,923 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [669ef94e8b26,46877,1731588865107] 2024-11-14T12:54:34,938 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/669ef94e8b26,46877,1731588865107 already deleted, retry=false 2024-11-14T12:54:34,938 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 669ef94e8b26,46877,1731588865107 expired; onlineServers=1 2024-11-14T12:54:34,948 INFO [regionserver/669ef94e8b26:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-14T12:54:34,949 INFO [regionserver/669ef94e8b26:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-14T12:54:35,013 DEBUG [RS:0;669ef94e8b26:43475 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T12:54:35,013 INFO [RS:0;669ef94e8b26:43475 {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T12:54:35,013 INFO [RS:0;669ef94e8b26:43475 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T12:54:35,014 INFO [RS:0;669ef94e8b26:43475 {}] hbase.ChoreService(370): Chore service for: regionserver/669ef94e8b26:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-14T12:54:35,014 INFO [RS:0;669ef94e8b26:43475 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T12:54:35,014 INFO [regionserver/669ef94e8b26:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T12:54:35,014 INFO [RS:0;669ef94e8b26:43475 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:43475 2024-11-14T12:54:35,022 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45309-0x1013ab2e72b0000, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T12:54:35,022 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43475-0x1013ab2e72b0001, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/669ef94e8b26,43475,1731588864980 2024-11-14T12:54:35,022 INFO [RS:0;669ef94e8b26:43475 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T12:54:35,023 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [669ef94e8b26,43475,1731588864980] 2024-11-14T12:54:35,030 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46877-0x1013ab2e72b0002, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T12:54:35,030 INFO [RS:1;669ef94e8b26:46877 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T12:54:35,030 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46877-0x1013ab2e72b0002, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T12:54:35,030 INFO [RS:1;669ef94e8b26:46877 {}] regionserver.HRegionServer(1031): Exiting; stopping=669ef94e8b26,46877,1731588865107; zookeeper connection closed. 2024-11-14T12:54:35,031 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@67273188 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@67273188 2024-11-14T12:54:35,038 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/669ef94e8b26,43475,1731588864980 already deleted, retry=false 2024-11-14T12:54:35,038 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 669ef94e8b26,43475,1731588864980 expired; onlineServers=0 2024-11-14T12:54:35,039 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '669ef94e8b26,45309,1731588864185' ***** 2024-11-14T12:54:35,039 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-14T12:54:35,039 INFO [M:0;669ef94e8b26:45309 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T12:54:35,039 INFO [M:0;669ef94e8b26:45309 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T12:54:35,039 DEBUG [M:0;669ef94e8b26:45309 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-14T12:54:35,040 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-14T12:54:35,040 DEBUG [M:0;669ef94e8b26:45309 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-14T12:54:35,040 DEBUG [master/669ef94e8b26:0:becomeActiveMaster-HFileCleaner.small.0-1731588866610 {}] cleaner.HFileCleaner(306): Exit Thread[master/669ef94e8b26:0:becomeActiveMaster-HFileCleaner.small.0-1731588866610,5,FailOnTimeoutGroup] 2024-11-14T12:54:35,040 DEBUG [master/669ef94e8b26:0:becomeActiveMaster-HFileCleaner.large.0-1731588866601 {}] cleaner.HFileCleaner(306): Exit Thread[master/669ef94e8b26:0:becomeActiveMaster-HFileCleaner.large.0-1731588866601,5,FailOnTimeoutGroup] 2024-11-14T12:54:35,040 INFO [M:0;669ef94e8b26:45309 {}] hbase.ChoreService(370): Chore service for: master/669ef94e8b26:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-14T12:54:35,040 INFO [M:0;669ef94e8b26:45309 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T12:54:35,040 DEBUG [M:0;669ef94e8b26:45309 {}] master.HMaster(1795): Stopping service threads 2024-11-14T12:54:35,040 INFO [M:0;669ef94e8b26:45309 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-14T12:54:35,040 INFO [M:0;669ef94e8b26:45309 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T12:54:35,040 INFO [M:0;669ef94e8b26:45309 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T12:54:35,041 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-14T12:54:35,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45309-0x1013ab2e72b0000, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-14T12:54:35,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45309-0x1013ab2e72b0000, quorum=127.0.0.1:60896, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T12:54:35,047 DEBUG [M:0;669ef94e8b26:45309 {}] zookeeper.ZKUtil(347): master:45309-0x1013ab2e72b0000, quorum=127.0.0.1:60896, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-14T12:54:35,047 WARN [M:0;669ef94e8b26:45309 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-14T12:54:35,047 INFO [M:0;669ef94e8b26:45309 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/.lastflushedseqids 2024-11-14T12:54:35,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36021 is added to blk_1073741870_1046 (size=1) 2024-11-14T12:54:35,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741870_1046 (size=1) 2024-11-14T12:54:35,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43871 is added to blk_1073741870_1046 (size=1) 2024-11-14T12:54:35,058 INFO [M:0;669ef94e8b26:45309 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T12:54:35,058 INFO [M:0;669ef94e8b26:45309 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-14T12:54:35,059 DEBUG [M:0;669ef94e8b26:45309 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-14T12:54:35,059 INFO [M:0;669ef94e8b26:45309 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T12:54:35,059 DEBUG [M:0;669ef94e8b26:45309 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T12:54:35,059 DEBUG [M:0;669ef94e8b26:45309 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-14T12:54:35,059 DEBUG [M:0;669ef94e8b26:45309 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T12:54:35,059 INFO [M:0;669ef94e8b26:45309 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=259.11 KB heapSize=314.92 KB 2024-11-14T12:54:35,074 DEBUG [M:0;669ef94e8b26:45309 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9b9b55db34ae48aeafd7ed2f88daa66c is 82, key is hbase:meta,,1/info:regioninfo/1731588873065/Put/seqid=0 2024-11-14T12:54:35,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35109 is added to blk_1073741871_1047 (size=6071) 2024-11-14T12:54:35,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43871 is added to blk_1073741871_1047 (size=6071) 2024-11-14T12:54:35,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36021 is added to blk_1073741871_1047 (size=6071) 2024-11-14T12:54:35,081 INFO [M:0;669ef94e8b26:45309 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=598 (bloomFilter=true), to=hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9b9b55db34ae48aeafd7ed2f88daa66c 2024-11-14T12:54:35,112 DEBUG [M:0;669ef94e8b26:45309 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36759/user/jenkins/test-data/5764751a-0b7e-c44d-dedb-b1851b430625/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5340bdfd2602464f96554c0818f330fb is 2984, key is \x00\x00\x00\x00\x00\x00\x00