2024-12-02 21:14:31,322 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-12-02 21:14:31,333 main DEBUG Took 0.009397 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-02 21:14:31,333 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-02 21:14:31,334 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-02 21:14:31,335 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-02 21:14:31,336 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 21:14:31,344 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-02 21:14:31,357 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 21:14:31,359 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 21:14:31,359 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 21:14:31,359 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 21:14:31,360 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 21:14:31,360 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 21:14:31,361 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 21:14:31,361 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 21:14:31,361 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 21:14:31,361 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 21:14:31,362 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 21:14:31,362 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 21:14:31,363 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 21:14:31,363 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 21:14:31,363 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 21:14:31,364 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 21:14:31,364 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 21:14:31,364 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 21:14:31,365 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 21:14:31,365 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 21:14:31,365 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 21:14:31,365 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 21:14:31,366 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 21:14:31,366 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 21:14:31,367 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 21:14:31,367 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-02 21:14:31,369 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 21:14:31,370 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-02 21:14:31,372 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-02 21:14:31,372 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-02 21:14:31,374 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-02 21:14:31,374 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-02 21:14:31,385 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-02 21:14:31,390 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-02 21:14:31,391 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-02 21:14:31,392 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-02 21:14:31,392 main DEBUG createAppenders(={Console}) 2024-12-02 21:14:31,393 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 initialized 2024-12-02 21:14:31,393 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-12-02 21:14:31,394 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 OK. 2024-12-02 21:14:31,394 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-02 21:14:31,395 main DEBUG OutputStream closed 2024-12-02 21:14:31,395 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-02 21:14:31,396 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-02 21:14:31,396 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@c88a337 OK 2024-12-02 21:14:31,469 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-02 21:14:31,471 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-02 21:14:31,473 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-02 21:14:31,474 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-02 21:14:31,474 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-02 21:14:31,475 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-02 21:14:31,475 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-02 21:14:31,476 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-02 21:14:31,476 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-02 21:14:31,476 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-02 21:14:31,477 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-02 21:14:31,477 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-02 21:14:31,478 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-02 21:14:31,478 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-02 21:14:31,478 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-02 21:14:31,479 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-02 21:14:31,479 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-02 21:14:31,480 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-02 21:14:31,482 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-02 21:14:31,483 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@13330ac6) with optional ClassLoader: null 2024-12-02 21:14:31,483 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-02 21:14:31,484 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@13330ac6] started OK. 2024-12-02T21:14:31,496 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP timeout: 13 mins 2024-12-02 21:14:31,499 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-02 21:14:31,499 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-02T21:14:31,697 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ddc45ef-3abe-1b20-1beb-1116b116a2b6 2024-12-02T21:14:31,716 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$HMasterForTest, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-02T21:14:31,731 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ddc45ef-3abe-1b20-1beb-1116b116a2b6/cluster_7b670c7c-ba39-4fed-b4f6-b61a91ca89dd, deleteOnExit=true 2024-12-02T21:14:31,731 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-02T21:14:31,732 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ddc45ef-3abe-1b20-1beb-1116b116a2b6/test.cache.data in system properties and HBase conf 2024-12-02T21:14:31,732 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ddc45ef-3abe-1b20-1beb-1116b116a2b6/hadoop.tmp.dir in system properties and HBase conf 2024-12-02T21:14:31,733 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ddc45ef-3abe-1b20-1beb-1116b116a2b6/hadoop.log.dir in system properties and HBase conf 2024-12-02T21:14:31,733 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ddc45ef-3abe-1b20-1beb-1116b116a2b6/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-02T21:14:31,734 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ddc45ef-3abe-1b20-1beb-1116b116a2b6/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-02T21:14:31,734 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-02T21:14:31,835 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-02T21:14:31,956 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-02T21:14:31,961 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ddc45ef-3abe-1b20-1beb-1116b116a2b6/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-02T21:14:31,962 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ddc45ef-3abe-1b20-1beb-1116b116a2b6/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-02T21:14:31,963 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ddc45ef-3abe-1b20-1beb-1116b116a2b6/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-02T21:14:31,963 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ddc45ef-3abe-1b20-1beb-1116b116a2b6/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-02T21:14:31,964 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ddc45ef-3abe-1b20-1beb-1116b116a2b6/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-02T21:14:31,965 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ddc45ef-3abe-1b20-1beb-1116b116a2b6/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-02T21:14:31,965 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ddc45ef-3abe-1b20-1beb-1116b116a2b6/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-02T21:14:31,966 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ddc45ef-3abe-1b20-1beb-1116b116a2b6/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-02T21:14:31,967 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ddc45ef-3abe-1b20-1beb-1116b116a2b6/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-02T21:14:31,967 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ddc45ef-3abe-1b20-1beb-1116b116a2b6/nfs.dump.dir in system properties and HBase conf 2024-12-02T21:14:31,968 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ddc45ef-3abe-1b20-1beb-1116b116a2b6/java.io.tmpdir in system properties and HBase conf 2024-12-02T21:14:31,968 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ddc45ef-3abe-1b20-1beb-1116b116a2b6/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-02T21:14:31,969 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ddc45ef-3abe-1b20-1beb-1116b116a2b6/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-02T21:14:31,970 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ddc45ef-3abe-1b20-1beb-1116b116a2b6/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-02T21:14:32,992 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-02T21:14:33,065 INFO [Time-limited test {}] log.Log(170): Logging initialized @2473ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-02T21:14:33,131 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T21:14:33,189 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-02T21:14:33,215 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-02T21:14:33,215 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-02T21:14:33,217 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-02T21:14:33,235 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T21:14:33,238 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@265b24f1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ddc45ef-3abe-1b20-1beb-1116b116a2b6/hadoop.log.dir/,AVAILABLE} 2024-12-02T21:14:33,240 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@76beb053{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-02T21:14:33,449 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@376fa8f3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ddc45ef-3abe-1b20-1beb-1116b116a2b6/java.io.tmpdir/jetty-localhost-40801-hadoop-hdfs-3_4_1-tests_jar-_-any-16249193035791297083/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-02T21:14:33,462 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@672b88ea{HTTP/1.1, (http/1.1)}{localhost:40801} 2024-12-02T21:14:33,462 INFO [Time-limited test {}] server.Server(415): Started @2871ms 2024-12-02T21:14:33,989 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T21:14:33,995 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-02T21:14:33,996 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-02T21:14:33,996 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-02T21:14:33,996 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-02T21:14:33,997 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@739a104e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ddc45ef-3abe-1b20-1beb-1116b116a2b6/hadoop.log.dir/,AVAILABLE} 2024-12-02T21:14:33,997 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1c5d2af1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-02T21:14:34,109 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2226d6a6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ddc45ef-3abe-1b20-1beb-1116b116a2b6/java.io.tmpdir/jetty-localhost-44067-hadoop-hdfs-3_4_1-tests_jar-_-any-9202987081054799072/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T21:14:34,110 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1da1f773{HTTP/1.1, (http/1.1)}{localhost:44067} 2024-12-02T21:14:34,110 INFO [Time-limited test {}] server.Server(415): Started @3519ms 2024-12-02T21:14:34,163 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-02T21:14:34,274 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T21:14:34,283 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-02T21:14:34,286 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-02T21:14:34,286 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-02T21:14:34,286 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-02T21:14:34,288 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@34a3e1ff{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ddc45ef-3abe-1b20-1beb-1116b116a2b6/hadoop.log.dir/,AVAILABLE} 2024-12-02T21:14:34,289 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@12f16529{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-02T21:14:34,389 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7cfccf27{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ddc45ef-3abe-1b20-1beb-1116b116a2b6/java.io.tmpdir/jetty-localhost-38875-hadoop-hdfs-3_4_1-tests_jar-_-any-10301518649214140351/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T21:14:34,390 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6c41bbb7{HTTP/1.1, (http/1.1)}{localhost:38875} 2024-12-02T21:14:34,390 INFO [Time-limited test {}] server.Server(415): Started @3799ms 2024-12-02T21:14:34,392 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-02T21:14:34,421 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T21:14:34,425 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-02T21:14:34,428 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-02T21:14:34,428 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-02T21:14:34,428 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-02T21:14:34,429 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@11ebaa12{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ddc45ef-3abe-1b20-1beb-1116b116a2b6/hadoop.log.dir/,AVAILABLE} 2024-12-02T21:14:34,429 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@52cf89e6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-02T21:14:34,533 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@67ddfd25{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ddc45ef-3abe-1b20-1beb-1116b116a2b6/java.io.tmpdir/jetty-localhost-34949-hadoop-hdfs-3_4_1-tests_jar-_-any-2864147706868384338/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T21:14:34,533 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@30e38263{HTTP/1.1, (http/1.1)}{localhost:34949} 2024-12-02T21:14:34,534 INFO [Time-limited test {}] server.Server(415): Started @3942ms 2024-12-02T21:14:34,535 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-02T21:14:36,319 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ddc45ef-3abe-1b20-1beb-1116b116a2b6/cluster_7b670c7c-ba39-4fed-b4f6-b61a91ca89dd/data/data3/current/BP-89888046-172.17.0.3-1733174072487/current, will proceed with Du for space computation calculation, 2024-12-02T21:14:36,319 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ddc45ef-3abe-1b20-1beb-1116b116a2b6/cluster_7b670c7c-ba39-4fed-b4f6-b61a91ca89dd/data/data4/current/BP-89888046-172.17.0.3-1733174072487/current, will proceed with Du for space computation calculation, 2024-12-02T21:14:36,320 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ddc45ef-3abe-1b20-1beb-1116b116a2b6/cluster_7b670c7c-ba39-4fed-b4f6-b61a91ca89dd/data/data1/current/BP-89888046-172.17.0.3-1733174072487/current, will proceed with Du for space computation calculation, 2024-12-02T21:14:36,320 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ddc45ef-3abe-1b20-1beb-1116b116a2b6/cluster_7b670c7c-ba39-4fed-b4f6-b61a91ca89dd/data/data2/current/BP-89888046-172.17.0.3-1733174072487/current, will proceed with Du for space computation calculation, 2024-12-02T21:14:36,342 WARN [Thread-132 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ddc45ef-3abe-1b20-1beb-1116b116a2b6/cluster_7b670c7c-ba39-4fed-b4f6-b61a91ca89dd/data/data5/current/BP-89888046-172.17.0.3-1733174072487/current, will proceed with Du for space computation calculation, 2024-12-02T21:14:36,343 WARN [Thread-133 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ddc45ef-3abe-1b20-1beb-1116b116a2b6/cluster_7b670c7c-ba39-4fed-b4f6-b61a91ca89dd/data/data6/current/BP-89888046-172.17.0.3-1733174072487/current, will proceed with Du for space computation calculation, 2024-12-02T21:14:36,369 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-02T21:14:36,369 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-02T21:14:36,369 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-02T21:14:36,425 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa5a8229b027a5a32 with lease ID 0x2fcceefdac1d7418: Processing first storage report for DS-9aa11248-a7e5-4660-b440-2c286bf86b1b from datanode DatanodeRegistration(127.0.0.1:42827, datanodeUuid=d519b3a5-8a9c-4043-807d-6a46a7dd91bb, infoPort=34107, infoSecurePort=0, ipcPort=42389, storageInfo=lv=-57;cid=testClusterID;nsid=726946471;c=1733174072487) 2024-12-02T21:14:36,426 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa5a8229b027a5a32 with lease ID 0x2fcceefdac1d7418: from storage DS-9aa11248-a7e5-4660-b440-2c286bf86b1b node DatanodeRegistration(127.0.0.1:42827, datanodeUuid=d519b3a5-8a9c-4043-807d-6a46a7dd91bb, infoPort=34107, infoSecurePort=0, ipcPort=42389, storageInfo=lv=-57;cid=testClusterID;nsid=726946471;c=1733174072487), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-12-02T21:14:36,427 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1586710607a218b4 with lease ID 0x2fcceefdac1d741a: Processing first storage report for DS-2d489d52-306c-448f-8298-290ac4052947 from datanode DatanodeRegistration(127.0.0.1:36203, datanodeUuid=36e36f92-7ceb-4c09-ae0c-d31d31deac21, infoPort=46475, infoSecurePort=0, ipcPort=44031, storageInfo=lv=-57;cid=testClusterID;nsid=726946471;c=1733174072487) 2024-12-02T21:14:36,427 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1586710607a218b4 with lease ID 0x2fcceefdac1d741a: from storage DS-2d489d52-306c-448f-8298-290ac4052947 node DatanodeRegistration(127.0.0.1:36203, datanodeUuid=36e36f92-7ceb-4c09-ae0c-d31d31deac21, infoPort=46475, infoSecurePort=0, ipcPort=44031, storageInfo=lv=-57;cid=testClusterID;nsid=726946471;c=1733174072487), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-02T21:14:36,427 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa5a8229b027a5a32 with lease ID 0x2fcceefdac1d7418: Processing first storage report for DS-5a4cba7c-b81c-4651-a835-654e20e72c0b from datanode DatanodeRegistration(127.0.0.1:42827, datanodeUuid=d519b3a5-8a9c-4043-807d-6a46a7dd91bb, infoPort=34107, infoSecurePort=0, ipcPort=42389, storageInfo=lv=-57;cid=testClusterID;nsid=726946471;c=1733174072487) 2024-12-02T21:14:36,428 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa5a8229b027a5a32 with lease ID 0x2fcceefdac1d7418: from storage DS-5a4cba7c-b81c-4651-a835-654e20e72c0b node DatanodeRegistration(127.0.0.1:42827, datanodeUuid=d519b3a5-8a9c-4043-807d-6a46a7dd91bb, infoPort=34107, infoSecurePort=0, ipcPort=42389, storageInfo=lv=-57;cid=testClusterID;nsid=726946471;c=1733174072487), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-02T21:14:36,428 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1586710607a218b4 with lease ID 0x2fcceefdac1d741a: Processing first storage report for DS-4ba17070-68dc-4647-ac73-aefb43f7f9bf from datanode DatanodeRegistration(127.0.0.1:36203, datanodeUuid=36e36f92-7ceb-4c09-ae0c-d31d31deac21, infoPort=46475, infoSecurePort=0, ipcPort=44031, storageInfo=lv=-57;cid=testClusterID;nsid=726946471;c=1733174072487) 2024-12-02T21:14:36,428 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1586710607a218b4 with lease ID 0x2fcceefdac1d741a: from storage DS-4ba17070-68dc-4647-ac73-aefb43f7f9bf node DatanodeRegistration(127.0.0.1:36203, datanodeUuid=36e36f92-7ceb-4c09-ae0c-d31d31deac21, infoPort=46475, infoSecurePort=0, ipcPort=44031, storageInfo=lv=-57;cid=testClusterID;nsid=726946471;c=1733174072487), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-02T21:14:36,429 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x615d6d587db9be43 with lease ID 0x2fcceefdac1d7419: Processing first storage report for DS-54bbf138-7100-4d62-a6b6-08df6ef7aa28 from datanode DatanodeRegistration(127.0.0.1:40959, datanodeUuid=108d404e-9f08-438e-8cee-7de8a8e4cbf3, infoPort=35813, infoSecurePort=0, ipcPort=38625, storageInfo=lv=-57;cid=testClusterID;nsid=726946471;c=1733174072487) 2024-12-02T21:14:36,429 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x615d6d587db9be43 with lease ID 0x2fcceefdac1d7419: from storage DS-54bbf138-7100-4d62-a6b6-08df6ef7aa28 node DatanodeRegistration(127.0.0.1:40959, datanodeUuid=108d404e-9f08-438e-8cee-7de8a8e4cbf3, infoPort=35813, infoSecurePort=0, ipcPort=38625, storageInfo=lv=-57;cid=testClusterID;nsid=726946471;c=1733174072487), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-02T21:14:36,430 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x615d6d587db9be43 with lease ID 0x2fcceefdac1d7419: Processing first storage report for DS-b93e555f-379c-47d7-888e-43600124dc2b from datanode DatanodeRegistration(127.0.0.1:40959, datanodeUuid=108d404e-9f08-438e-8cee-7de8a8e4cbf3, infoPort=35813, infoSecurePort=0, ipcPort=38625, storageInfo=lv=-57;cid=testClusterID;nsid=726946471;c=1733174072487) 2024-12-02T21:14:36,430 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x615d6d587db9be43 with lease ID 0x2fcceefdac1d7419: from storage DS-b93e555f-379c-47d7-888e-43600124dc2b node DatanodeRegistration(127.0.0.1:40959, datanodeUuid=108d404e-9f08-438e-8cee-7de8a8e4cbf3, infoPort=35813, infoSecurePort=0, ipcPort=38625, storageInfo=lv=-57;cid=testClusterID;nsid=726946471;c=1733174072487), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-02T21:14:36,457 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ddc45ef-3abe-1b20-1beb-1116b116a2b6 2024-12-02T21:14:36,527 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ddc45ef-3abe-1b20-1beb-1116b116a2b6/cluster_7b670c7c-ba39-4fed-b4f6-b61a91ca89dd/zookeeper_0, clientPort=63400, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ddc45ef-3abe-1b20-1beb-1116b116a2b6/cluster_7b670c7c-ba39-4fed-b4f6-b61a91ca89dd/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6ddc45ef-3abe-1b20-1beb-1116b116a2b6/cluster_7b670c7c-ba39-4fed-b4f6-b61a91ca89dd/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-02T21:14:36,535 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=63400 2024-12-02T21:14:36,544 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T21:14:36,547 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T21:14:36,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42827 is added to blk_1073741825_1001 (size=7) 2024-12-02T21:14:36,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741825_1001 (size=7) 2024-12-02T21:14:36,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36203 is added to blk_1073741825_1001 (size=7) 2024-12-02T21:14:37,181 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b with version=8 2024-12-02T21:14:37,182 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/hbase-staging 2024-12-02T21:14:37,253 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-02T21:14:37,440 INFO [Time-limited test {}] client.ConnectionUtils(128): master/87c3fdb6c570:0 server-side Connection retries=45 2024-12-02T21:14:37,449 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T21:14:37,450 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-02T21:14:37,454 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-02T21:14:37,454 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T21:14:37,454 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-02T21:14:37,595 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-02T21:14:37,653 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-02T21:14:37,660 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-02T21:14:37,663 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-02T21:14:37,685 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 2479 (auto-detected) 2024-12-02T21:14:37,685 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-12-02T21:14:37,701 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:42045 2024-12-02T21:14:37,720 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42045 connecting to ZooKeeper ensemble=127.0.0.1:63400 2024-12-02T21:14:37,850 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:420450x0, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-02T21:14:37,853 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42045-0x10197e5a0060000 connected 2024-12-02T21:14:37,943 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T21:14:37,949 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T21:14:37,961 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42045-0x10197e5a0060000, quorum=127.0.0.1:63400, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T21:14:37,967 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b, hbase.cluster.distributed=false 2024-12-02T21:14:37,993 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42045-0x10197e5a0060000, quorum=127.0.0.1:63400, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-02T21:14:37,998 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42045 2024-12-02T21:14:37,998 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42045 2024-12-02T21:14:37,999 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42045 2024-12-02T21:14:37,999 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42045 2024-12-02T21:14:37,999 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42045 2024-12-02T21:14:38,091 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/87c3fdb6c570:0 server-side Connection retries=45 2024-12-02T21:14:38,093 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T21:14:38,093 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-02T21:14:38,093 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-02T21:14:38,093 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T21:14:38,094 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-02T21:14:38,096 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-02T21:14:38,098 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-02T21:14:38,098 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:40413 2024-12-02T21:14:38,100 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40413 connecting to ZooKeeper ensemble=127.0.0.1:63400 2024-12-02T21:14:38,101 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T21:14:38,104 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T21:14:38,143 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:404130x0, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-02T21:14:38,144 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:404130x0, quorum=127.0.0.1:63400, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T21:14:38,144 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40413-0x10197e5a0060001 connected 2024-12-02T21:14:38,148 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-02T21:14:38,157 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-02T21:14:38,159 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40413-0x10197e5a0060001, quorum=127.0.0.1:63400, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-02T21:14:38,164 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40413-0x10197e5a0060001, quorum=127.0.0.1:63400, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-02T21:14:38,165 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40413 2024-12-02T21:14:38,165 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40413 2024-12-02T21:14:38,165 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40413 2024-12-02T21:14:38,166 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40413 2024-12-02T21:14:38,166 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40413 2024-12-02T21:14:38,183 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/87c3fdb6c570:0 server-side Connection retries=45 2024-12-02T21:14:38,183 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T21:14:38,183 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-02T21:14:38,184 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-02T21:14:38,184 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T21:14:38,184 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-02T21:14:38,184 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-02T21:14:38,185 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-02T21:14:38,186 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:36585 2024-12-02T21:14:38,187 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36585 connecting to ZooKeeper ensemble=127.0.0.1:63400 2024-12-02T21:14:38,188 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T21:14:38,191 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T21:14:38,271 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:365850x0, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-02T21:14:38,272 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:365850x0, quorum=127.0.0.1:63400, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T21:14:38,273 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36585-0x10197e5a0060002 connected 2024-12-02T21:14:38,273 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-02T21:14:38,274 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-02T21:14:38,276 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36585-0x10197e5a0060002, quorum=127.0.0.1:63400, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-02T21:14:38,278 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36585-0x10197e5a0060002, quorum=127.0.0.1:63400, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-02T21:14:38,278 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36585 2024-12-02T21:14:38,279 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36585 2024-12-02T21:14:38,279 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36585 2024-12-02T21:14:38,280 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36585 2024-12-02T21:14:38,280 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36585 2024-12-02T21:14:38,299 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/87c3fdb6c570:0 server-side Connection retries=45 2024-12-02T21:14:38,299 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T21:14:38,299 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-02T21:14:38,300 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-02T21:14:38,300 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T21:14:38,300 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-02T21:14:38,300 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-02T21:14:38,300 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-02T21:14:38,301 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:33459 2024-12-02T21:14:38,303 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33459 connecting to ZooKeeper ensemble=127.0.0.1:63400 2024-12-02T21:14:38,305 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T21:14:38,309 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T21:14:38,365 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:334590x0, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-02T21:14:38,367 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:334590x0, quorum=127.0.0.1:63400, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T21:14:38,367 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33459-0x10197e5a0060003 connected 2024-12-02T21:14:38,368 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-02T21:14:38,370 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-02T21:14:38,372 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33459-0x10197e5a0060003, quorum=127.0.0.1:63400, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-02T21:14:38,375 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33459-0x10197e5a0060003, quorum=127.0.0.1:63400, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-02T21:14:38,376 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33459 2024-12-02T21:14:38,376 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33459 2024-12-02T21:14:38,377 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33459 2024-12-02T21:14:38,377 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33459 2024-12-02T21:14:38,378 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33459 2024-12-02T21:14:38,395 DEBUG [M:0;87c3fdb6c570:42045 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;87c3fdb6c570:42045 2024-12-02T21:14:38,396 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/87c3fdb6c570,42045,1733174077295 2024-12-02T21:14:38,425 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33459-0x10197e5a0060003, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T21:14:38,425 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40413-0x10197e5a0060001, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T21:14:38,426 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36585-0x10197e5a0060002, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T21:14:38,426 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42045-0x10197e5a0060000, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T21:14:38,428 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42045-0x10197e5a0060000, quorum=127.0.0.1:63400, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/87c3fdb6c570,42045,1733174077295 2024-12-02T21:14:38,456 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40413-0x10197e5a0060001, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-02T21:14:38,456 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33459-0x10197e5a0060003, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-02T21:14:38,456 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40413-0x10197e5a0060001, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T21:14:38,456 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42045-0x10197e5a0060000, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T21:14:38,456 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36585-0x10197e5a0060002, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-02T21:14:38,456 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33459-0x10197e5a0060003, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T21:14:38,456 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36585-0x10197e5a0060002, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T21:14:38,459 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42045-0x10197e5a0060000, quorum=127.0.0.1:63400, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-02T21:14:38,460 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/87c3fdb6c570,42045,1733174077295 from backup master directory 2024-12-02T21:14:38,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42045-0x10197e5a0060000, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/87c3fdb6c570,42045,1733174077295 2024-12-02T21:14:38,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40413-0x10197e5a0060001, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T21:14:38,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36585-0x10197e5a0060002, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T21:14:38,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33459-0x10197e5a0060003, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T21:14:38,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42045-0x10197e5a0060000, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T21:14:38,474 WARN [master/87c3fdb6c570:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-02T21:14:38,474 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=87c3fdb6c570,42045,1733174077295 2024-12-02T21:14:38,476 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-02T21:14:38,477 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-02T21:14:38,529 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/hbase.id] with ID: a3fb6e34-ca27-4453-9e72-69b912f26690 2024-12-02T21:14:38,529 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/.tmp/hbase.id 2024-12-02T21:14:38,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36203 is added to blk_1073741826_1002 (size=42) 2024-12-02T21:14:38,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42827 is added to blk_1073741826_1002 (size=42) 2024-12-02T21:14:38,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741826_1002 (size=42) 2024-12-02T21:14:38,560 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/.tmp/hbase.id]:[hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/hbase.id] 2024-12-02T21:14:38,624 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T21:14:38,630 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-02T21:14:38,646 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 14ms. 2024-12-02T21:14:38,706 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33459-0x10197e5a0060003, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T21:14:38,706 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40413-0x10197e5a0060001, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T21:14:38,707 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36585-0x10197e5a0060002, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T21:14:38,707 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42045-0x10197e5a0060000, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T21:14:38,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36203 is added to blk_1073741827_1003 (size=196) 2024-12-02T21:14:38,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741827_1003 (size=196) 2024-12-02T21:14:38,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42827 is added to blk_1073741827_1003 (size=196) 2024-12-02T21:14:38,743 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T21:14:38,745 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-02T21:14:38,758 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:14:38,762 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T21:14:38,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741828_1004 (size=1189) 2024-12-02T21:14:38,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42827 is added to blk_1073741828_1004 (size=1189) 2024-12-02T21:14:38,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36203 is added to blk_1073741828_1004 (size=1189) 2024-12-02T21:14:38,807 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/MasterData/data/master/store 2024-12-02T21:14:38,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741829_1005 (size=34) 2024-12-02T21:14:38,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36203 is added to blk_1073741829_1005 (size=34) 2024-12-02T21:14:38,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42827 is added to blk_1073741829_1005 (size=34) 2024-12-02T21:14:38,843 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-02T21:14:38,848 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:14:38,850 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-02T21:14:38,850 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T21:14:38,850 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T21:14:38,852 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-02T21:14:38,853 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T21:14:38,853 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T21:14:38,855 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733174078849Disabling compacts and flushes for region at 1733174078849Disabling writes for close at 1733174078853 (+4 ms)Writing region close event to WAL at 1733174078853Closed at 1733174078853 2024-12-02T21:14:38,858 WARN [master/87c3fdb6c570:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/MasterData/data/master/store/.initializing 2024-12-02T21:14:38,858 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/MasterData/WALs/87c3fdb6c570,42045,1733174077295 2024-12-02T21:14:38,867 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-02T21:14:38,885 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=87c3fdb6c570%2C42045%2C1733174077295, suffix=, logDir=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/MasterData/WALs/87c3fdb6c570,42045,1733174077295, archiveDir=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/MasterData/oldWALs, maxLogs=10 2024-12-02T21:14:38,915 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/MasterData/WALs/87c3fdb6c570,42045,1733174077295/87c3fdb6c570%2C42045%2C1733174077295.1733174078891, exclude list is [], retry=0 2024-12-02T21:14:38,933 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36203,DS-2d489d52-306c-448f-8298-290ac4052947,DISK] 2024-12-02T21:14:38,933 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42827,DS-9aa11248-a7e5-4660-b440-2c286bf86b1b,DISK] 2024-12-02T21:14:38,933 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40959,DS-54bbf138-7100-4d62-a6b6-08df6ef7aa28,DISK] 2024-12-02T21:14:38,936 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-02T21:14:38,970 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/MasterData/WALs/87c3fdb6c570,42045,1733174077295/87c3fdb6c570%2C42045%2C1733174077295.1733174078891 2024-12-02T21:14:38,970 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35813:35813),(127.0.0.1/127.0.0.1:34107:34107),(127.0.0.1/127.0.0.1:46475:46475)] 2024-12-02T21:14:38,971 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-02T21:14:38,972 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:14:38,975 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T21:14:38,976 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T21:14:39,022 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T21:14:39,054 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-02T21:14:39,057 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:14:39,060 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T21:14:39,061 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T21:14:39,065 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-02T21:14:39,065 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:14:39,067 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T21:14:39,067 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T21:14:39,071 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-02T21:14:39,071 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:14:39,073 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T21:14:39,073 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T21:14:39,077 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-02T21:14:39,077 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:14:39,079 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T21:14:39,079 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T21:14:39,083 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-02T21:14:39,084 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-02T21:14:39,089 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T21:14:39,090 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T21:14:39,093 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-02T21:14:39,097 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T21:14:39,102 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T21:14:39,104 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65147509, jitterRate=-0.02922646701335907}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-02T21:14:39,110 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733174078989Initializing all the Stores at 1733174078991 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733174078992 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174078992Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174078993 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174078993Cleaning up temporary data from old regions at 1733174079090 (+97 ms)Region opened successfully at 1733174079109 (+19 ms) 2024-12-02T21:14:39,111 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-02T21:14:39,142 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@170731a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=87c3fdb6c570/172.17.0.3:0 2024-12-02T21:14:39,170 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-02T21:14:39,179 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-02T21:14:39,179 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-02T21:14:39,182 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-02T21:14:39,183 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-02T21:14:39,188 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-02T21:14:39,189 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-02T21:14:39,212 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-02T21:14:39,221 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42045-0x10197e5a0060000, quorum=127.0.0.1:63400, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-02T21:14:39,280 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-02T21:14:39,285 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-02T21:14:39,287 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42045-0x10197e5a0060000, quorum=127.0.0.1:63400, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-02T21:14:39,342 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-02T21:14:39,347 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-02T21:14:39,352 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42045-0x10197e5a0060000, quorum=127.0.0.1:63400, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-02T21:14:39,372 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-02T21:14:39,373 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42045-0x10197e5a0060000, quorum=127.0.0.1:63400, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-02T21:14:39,438 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-02T21:14:39,462 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42045-0x10197e5a0060000, quorum=127.0.0.1:63400, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-02T21:14:39,501 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-02T21:14:39,525 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40413-0x10197e5a0060001, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-02T21:14:39,525 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36585-0x10197e5a0060002, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-02T21:14:39,525 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42045-0x10197e5a0060000, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-02T21:14:39,525 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33459-0x10197e5a0060003, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-02T21:14:39,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40413-0x10197e5a0060001, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T21:14:39,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36585-0x10197e5a0060002, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T21:14:39,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42045-0x10197e5a0060000, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T21:14:39,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33459-0x10197e5a0060003, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T21:14:39,529 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=87c3fdb6c570,42045,1733174077295, sessionid=0x10197e5a0060000, setting cluster-up flag (Was=false) 2024-12-02T21:14:39,660 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33459-0x10197e5a0060003, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T21:14:39,660 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40413-0x10197e5a0060001, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T21:14:39,660 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36585-0x10197e5a0060002, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T21:14:39,660 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42045-0x10197e5a0060000, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T21:14:39,862 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-02T21:14:39,864 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=87c3fdb6c570,42045,1733174077295 2024-12-02T21:14:39,887 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42045-0x10197e5a0060000, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T21:14:39,887 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33459-0x10197e5a0060003, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T21:14:39,887 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40413-0x10197e5a0060001, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T21:14:39,887 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36585-0x10197e5a0060002, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T21:14:39,919 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-02T21:14:39,923 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=87c3fdb6c570,42045,1733174077295 2024-12-02T21:14:39,932 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-02T21:14:39,982 INFO [RS:0;87c3fdb6c570:40413 {}] regionserver.HRegionServer(746): ClusterId : a3fb6e34-ca27-4453-9e72-69b912f26690 2024-12-02T21:14:39,982 INFO [RS:2;87c3fdb6c570:33459 {}] regionserver.HRegionServer(746): ClusterId : a3fb6e34-ca27-4453-9e72-69b912f26690 2024-12-02T21:14:39,983 INFO [RS:1;87c3fdb6c570:36585 {}] regionserver.HRegionServer(746): ClusterId : a3fb6e34-ca27-4453-9e72-69b912f26690 2024-12-02T21:14:39,985 DEBUG [RS:1;87c3fdb6c570:36585 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-02T21:14:39,985 DEBUG [RS:0;87c3fdb6c570:40413 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-02T21:14:39,985 DEBUG [RS:2;87c3fdb6c570:33459 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-02T21:14:39,999 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-02T21:14:40,008 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-02T21:14:40,013 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-02T21:14:40,016 DEBUG [RS:0;87c3fdb6c570:40413 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-02T21:14:40,016 DEBUG [RS:2;87c3fdb6c570:33459 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-02T21:14:40,016 DEBUG [RS:1;87c3fdb6c570:36585 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-02T21:14:40,016 DEBUG [RS:0;87c3fdb6c570:40413 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-02T21:14:40,016 DEBUG [RS:2;87c3fdb6c570:33459 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-02T21:14:40,016 DEBUG [RS:1;87c3fdb6c570:36585 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-02T21:14:40,019 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 87c3fdb6c570,42045,1733174077295 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-02T21:14:40,036 DEBUG [RS:2;87c3fdb6c570:33459 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-02T21:14:40,036 DEBUG [RS:0;87c3fdb6c570:40413 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-02T21:14:40,037 DEBUG [RS:1;87c3fdb6c570:36585 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-02T21:14:40,037 DEBUG [RS:2;87c3fdb6c570:33459 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56801c79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=87c3fdb6c570/172.17.0.3:0 2024-12-02T21:14:40,037 DEBUG [RS:0;87c3fdb6c570:40413 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34dd8b0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=87c3fdb6c570/172.17.0.3:0 2024-12-02T21:14:40,037 DEBUG [RS:1;87c3fdb6c570:36585 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12b00ec1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=87c3fdb6c570/172.17.0.3:0 2024-12-02T21:14:40,039 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/87c3fdb6c570:0, corePoolSize=5, maxPoolSize=5 2024-12-02T21:14:40,040 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/87c3fdb6c570:0, corePoolSize=5, maxPoolSize=5 2024-12-02T21:14:40,040 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/87c3fdb6c570:0, corePoolSize=5, maxPoolSize=5 2024-12-02T21:14:40,041 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/87c3fdb6c570:0, corePoolSize=5, maxPoolSize=5 2024-12-02T21:14:40,041 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/87c3fdb6c570:0, corePoolSize=10, maxPoolSize=10 2024-12-02T21:14:40,041 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/87c3fdb6c570:0, corePoolSize=1, maxPoolSize=1 2024-12-02T21:14:40,041 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/87c3fdb6c570:0, corePoolSize=2, maxPoolSize=2 2024-12-02T21:14:40,041 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/87c3fdb6c570:0, corePoolSize=1, maxPoolSize=1 2024-12-02T21:14:40,044 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733174110044 2024-12-02T21:14:40,046 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-02T21:14:40,047 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-02T21:14:40,050 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-02T21:14:40,050 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-02T21:14:40,052 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-02T21:14:40,052 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-02T21:14:40,053 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-02T21:14:40,053 DEBUG [RS:2;87c3fdb6c570:33459 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;87c3fdb6c570:33459 2024-12-02T21:14:40,053 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-02T21:14:40,055 INFO [RS:2;87c3fdb6c570:33459 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-02T21:14:40,055 INFO [RS:2;87c3fdb6c570:33459 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-02T21:14:40,055 DEBUG [RS:2;87c3fdb6c570:33459 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-02T21:14:40,057 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:14:40,055 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-02T21:14:40,057 DEBUG [RS:1;87c3fdb6c570:36585 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;87c3fdb6c570:36585 2024-12-02T21:14:40,057 INFO [RS:1;87c3fdb6c570:36585 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-02T21:14:40,057 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-02T21:14:40,057 INFO [RS:1;87c3fdb6c570:36585 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-02T21:14:40,057 DEBUG [RS:1;87c3fdb6c570:36585 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-02T21:14:40,057 DEBUG [RS:0;87c3fdb6c570:40413 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;87c3fdb6c570:40413 2024-12-02T21:14:40,058 INFO [RS:0;87c3fdb6c570:40413 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-02T21:14:40,058 INFO [RS:0;87c3fdb6c570:40413 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-02T21:14:40,058 DEBUG [RS:0;87c3fdb6c570:40413 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-02T21:14:40,058 INFO [RS:2;87c3fdb6c570:33459 {}] regionserver.HRegionServer(2659): reportForDuty to master=87c3fdb6c570,42045,1733174077295 with port=33459, startcode=1733174078298 2024-12-02T21:14:40,059 INFO [RS:1;87c3fdb6c570:36585 {}] regionserver.HRegionServer(2659): reportForDuty to master=87c3fdb6c570,42045,1733174077295 with port=36585, startcode=1733174078182 2024-12-02T21:14:40,059 INFO [RS:0;87c3fdb6c570:40413 {}] regionserver.HRegionServer(2659): reportForDuty to master=87c3fdb6c570,42045,1733174077295 with port=40413, startcode=1733174078061 2024-12-02T21:14:40,059 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-02T21:14:40,060 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-02T21:14:40,060 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-02T21:14:40,062 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-02T21:14:40,063 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-02T21:14:40,064 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/87c3fdb6c570:0:becomeActiveMaster-HFileCleaner.large.0-1733174080064,5,FailOnTimeoutGroup] 2024-12-02T21:14:40,068 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/87c3fdb6c570:0:becomeActiveMaster-HFileCleaner.small.0-1733174080065,5,FailOnTimeoutGroup] 2024-12-02T21:14:40,068 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-02T21:14:40,069 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-02T21:14:40,070 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-02T21:14:40,071 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-02T21:14:40,072 DEBUG [RS:0;87c3fdb6c570:40413 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-02T21:14:40,072 DEBUG [RS:2;87c3fdb6c570:33459 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-02T21:14:40,072 DEBUG [RS:1;87c3fdb6c570:36585 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-02T21:14:40,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42827 is added to blk_1073741831_1007 (size=1321) 2024-12-02T21:14:40,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741831_1007 (size=1321) 2024-12-02T21:14:40,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36203 is added to blk_1073741831_1007 (size=1321) 2024-12-02T21:14:40,082 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-02T21:14:40,083 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b 2024-12-02T21:14:40,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36203 is added to blk_1073741832_1008 (size=32) 2024-12-02T21:14:40,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42827 is added to blk_1073741832_1008 (size=32) 2024-12-02T21:14:40,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741832_1008 (size=32) 2024-12-02T21:14:40,123 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:14:40,126 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-02T21:14:40,127 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37549, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-02T21:14:40,127 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44483, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-12-02T21:14:40,127 INFO [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42019, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-12-02T21:14:40,130 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-02T21:14:40,130 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:14:40,131 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T21:14:40,132 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-02T21:14:40,133 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42045 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:40,135 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-02T21:14:40,135 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:14:40,135 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42045 {}] master.ServerManager(517): Registering regionserver=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:40,136 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T21:14:40,136 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-02T21:14:40,140 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-02T21:14:40,140 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:14:40,141 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T21:14:40,141 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-02T21:14:40,144 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-02T21:14:40,145 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:14:40,146 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T21:14:40,147 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-02T21:14:40,147 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42045 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 87c3fdb6c570,33459,1733174078298 2024-12-02T21:14:40,147 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42045 {}] master.ServerManager(517): Registering regionserver=87c3fdb6c570,33459,1733174078298 2024-12-02T21:14:40,150 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/hbase/meta/1588230740 2024-12-02T21:14:40,151 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42045 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:40,151 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42045 {}] master.ServerManager(517): Registering regionserver=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:40,151 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/hbase/meta/1588230740 2024-12-02T21:14:40,153 DEBUG [RS:2;87c3fdb6c570:33459 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b 2024-12-02T21:14:40,153 DEBUG [RS:0;87c3fdb6c570:40413 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b 2024-12-02T21:14:40,153 DEBUG [RS:0;87c3fdb6c570:40413 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43251 2024-12-02T21:14:40,153 DEBUG [RS:2;87c3fdb6c570:33459 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43251 2024-12-02T21:14:40,153 DEBUG [RS:0;87c3fdb6c570:40413 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-02T21:14:40,153 DEBUG [RS:2;87c3fdb6c570:33459 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-02T21:14:40,155 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-02T21:14:40,155 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-02T21:14:40,155 DEBUG [RS:1;87c3fdb6c570:36585 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b 2024-12-02T21:14:40,156 DEBUG [RS:1;87c3fdb6c570:36585 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43251 2024-12-02T21:14:40,156 DEBUG [RS:1;87c3fdb6c570:36585 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-02T21:14:40,156 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-02T21:14:40,160 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-02T21:14:40,164 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T21:14:40,165 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68016350, jitterRate=0.013522595167160034}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-02T21:14:40,168 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733174080123Initializing all the Stores at 1733174080125 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733174080125Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733174080126 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174080126Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733174080126Cleaning up temporary data from old regions at 1733174080155 (+29 ms)Region opened successfully at 1733174080167 (+12 ms) 2024-12-02T21:14:40,168 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-02T21:14:40,168 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-02T21:14:40,168 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-02T21:14:40,168 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-02T21:14:40,168 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-02T21:14:40,170 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-02T21:14:40,170 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733174080168Disabling compacts and flushes for region at 1733174080168Disabling writes for close at 1733174080168Writing region close event to WAL at 1733174080169 (+1 ms)Closed at 1733174080170 (+1 ms) 2024-12-02T21:14:40,173 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-02T21:14:40,173 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-02T21:14:40,181 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-02T21:14:40,191 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T21:14:40,194 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-02T21:14:40,209 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42045-0x10197e5a0060000, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-02T21:14:40,231 DEBUG [RS:2;87c3fdb6c570:33459 {}] zookeeper.ZKUtil(111): regionserver:33459-0x10197e5a0060003, quorum=127.0.0.1:63400, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/87c3fdb6c570,33459,1733174078298 2024-12-02T21:14:40,231 DEBUG [RS:0;87c3fdb6c570:40413 {}] zookeeper.ZKUtil(111): regionserver:40413-0x10197e5a0060001, quorum=127.0.0.1:63400, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:40,231 WARN [RS:0;87c3fdb6c570:40413 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-02T21:14:40,231 WARN [RS:2;87c3fdb6c570:33459 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-02T21:14:40,232 INFO [RS:2;87c3fdb6c570:33459 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T21:14:40,232 INFO [RS:0;87c3fdb6c570:40413 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T21:14:40,232 DEBUG [RS:1;87c3fdb6c570:36585 {}] zookeeper.ZKUtil(111): regionserver:36585-0x10197e5a0060002, quorum=127.0.0.1:63400, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:40,232 WARN [RS:1;87c3fdb6c570:36585 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-02T21:14:40,232 INFO [RS:1;87c3fdb6c570:36585 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T21:14:40,232 DEBUG [RS:0;87c3fdb6c570:40413 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/WALs/87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:40,232 DEBUG [RS:2;87c3fdb6c570:33459 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/WALs/87c3fdb6c570,33459,1733174078298 2024-12-02T21:14:40,233 DEBUG [RS:1;87c3fdb6c570:36585 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/WALs/87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:40,234 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [87c3fdb6c570,33459,1733174078298] 2024-12-02T21:14:40,234 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [87c3fdb6c570,36585,1733174078182] 2024-12-02T21:14:40,234 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [87c3fdb6c570,40413,1733174078061] 2024-12-02T21:14:40,260 INFO [RS:2;87c3fdb6c570:33459 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-02T21:14:40,260 INFO [RS:0;87c3fdb6c570:40413 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-02T21:14:40,260 INFO [RS:1;87c3fdb6c570:36585 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-02T21:14:40,271 INFO [RS:1;87c3fdb6c570:36585 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-02T21:14:40,271 INFO [RS:0;87c3fdb6c570:40413 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-02T21:14:40,271 INFO [RS:2;87c3fdb6c570:33459 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-02T21:14:40,278 INFO [RS:0;87c3fdb6c570:40413 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-02T21:14:40,278 INFO [RS:1;87c3fdb6c570:36585 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-02T21:14:40,278 INFO [RS:2;87c3fdb6c570:33459 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-02T21:14:40,279 INFO [RS:2;87c3fdb6c570:33459 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T21:14:40,279 INFO [RS:0;87c3fdb6c570:40413 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T21:14:40,279 INFO [RS:1;87c3fdb6c570:36585 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T21:14:40,280 INFO [RS:2;87c3fdb6c570:33459 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-02T21:14:40,280 INFO [RS:0;87c3fdb6c570:40413 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-02T21:14:40,280 INFO [RS:1;87c3fdb6c570:36585 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-02T21:14:40,285 INFO [RS:1;87c3fdb6c570:36585 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-02T21:14:40,285 INFO [RS:2;87c3fdb6c570:33459 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-02T21:14:40,285 INFO [RS:0;87c3fdb6c570:40413 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-02T21:14:40,287 INFO [RS:1;87c3fdb6c570:36585 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-02T21:14:40,287 INFO [RS:2;87c3fdb6c570:33459 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-02T21:14:40,287 INFO [RS:0;87c3fdb6c570:40413 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-02T21:14:40,288 DEBUG [RS:1;87c3fdb6c570:36585 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/87c3fdb6c570:0, corePoolSize=1, maxPoolSize=1 2024-12-02T21:14:40,288 DEBUG [RS:0;87c3fdb6c570:40413 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/87c3fdb6c570:0, corePoolSize=1, maxPoolSize=1 2024-12-02T21:14:40,288 DEBUG [RS:2;87c3fdb6c570:33459 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/87c3fdb6c570:0, corePoolSize=1, maxPoolSize=1 2024-12-02T21:14:40,288 DEBUG [RS:1;87c3fdb6c570:36585 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/87c3fdb6c570:0, corePoolSize=1, maxPoolSize=1 2024-12-02T21:14:40,288 DEBUG [RS:2;87c3fdb6c570:33459 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/87c3fdb6c570:0, corePoolSize=1, maxPoolSize=1 2024-12-02T21:14:40,288 DEBUG [RS:0;87c3fdb6c570:40413 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/87c3fdb6c570:0, corePoolSize=1, maxPoolSize=1 2024-12-02T21:14:40,288 DEBUG [RS:1;87c3fdb6c570:36585 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/87c3fdb6c570:0, corePoolSize=1, maxPoolSize=1 2024-12-02T21:14:40,288 DEBUG [RS:0;87c3fdb6c570:40413 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/87c3fdb6c570:0, corePoolSize=1, maxPoolSize=1 2024-12-02T21:14:40,288 DEBUG [RS:2;87c3fdb6c570:33459 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/87c3fdb6c570:0, corePoolSize=1, maxPoolSize=1 2024-12-02T21:14:40,288 DEBUG [RS:1;87c3fdb6c570:36585 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/87c3fdb6c570:0, corePoolSize=1, maxPoolSize=1 2024-12-02T21:14:40,288 DEBUG [RS:0;87c3fdb6c570:40413 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/87c3fdb6c570:0, corePoolSize=1, maxPoolSize=1 2024-12-02T21:14:40,288 DEBUG [RS:2;87c3fdb6c570:33459 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/87c3fdb6c570:0, corePoolSize=1, maxPoolSize=1 2024-12-02T21:14:40,288 DEBUG [RS:1;87c3fdb6c570:36585 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/87c3fdb6c570:0, corePoolSize=1, maxPoolSize=1 2024-12-02T21:14:40,288 DEBUG [RS:2;87c3fdb6c570:33459 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/87c3fdb6c570:0, corePoolSize=1, maxPoolSize=1 2024-12-02T21:14:40,288 DEBUG [RS:0;87c3fdb6c570:40413 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/87c3fdb6c570:0, corePoolSize=1, maxPoolSize=1 2024-12-02T21:14:40,288 DEBUG [RS:1;87c3fdb6c570:36585 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/87c3fdb6c570:0, corePoolSize=2, maxPoolSize=2 2024-12-02T21:14:40,289 DEBUG [RS:2;87c3fdb6c570:33459 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/87c3fdb6c570:0, corePoolSize=2, maxPoolSize=2 2024-12-02T21:14:40,289 DEBUG [RS:0;87c3fdb6c570:40413 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/87c3fdb6c570:0, corePoolSize=2, maxPoolSize=2 2024-12-02T21:14:40,289 DEBUG [RS:1;87c3fdb6c570:36585 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0, corePoolSize=1, maxPoolSize=1 2024-12-02T21:14:40,289 DEBUG [RS:2;87c3fdb6c570:33459 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0, corePoolSize=1, maxPoolSize=1 2024-12-02T21:14:40,289 DEBUG [RS:0;87c3fdb6c570:40413 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/87c3fdb6c570:0, corePoolSize=1, maxPoolSize=1 2024-12-02T21:14:40,289 DEBUG [RS:1;87c3fdb6c570:36585 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/87c3fdb6c570:0, corePoolSize=1, maxPoolSize=1 2024-12-02T21:14:40,289 DEBUG [RS:2;87c3fdb6c570:33459 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/87c3fdb6c570:0, corePoolSize=1, maxPoolSize=1 2024-12-02T21:14:40,289 DEBUG [RS:0;87c3fdb6c570:40413 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/87c3fdb6c570:0, corePoolSize=1, maxPoolSize=1 2024-12-02T21:14:40,289 DEBUG [RS:0;87c3fdb6c570:40413 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/87c3fdb6c570:0, corePoolSize=1, maxPoolSize=1 2024-12-02T21:14:40,289 DEBUG [RS:1;87c3fdb6c570:36585 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/87c3fdb6c570:0, corePoolSize=1, maxPoolSize=1 2024-12-02T21:14:40,289 DEBUG [RS:2;87c3fdb6c570:33459 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/87c3fdb6c570:0, corePoolSize=1, maxPoolSize=1 2024-12-02T21:14:40,289 DEBUG [RS:0;87c3fdb6c570:40413 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/87c3fdb6c570:0, corePoolSize=1, maxPoolSize=1 2024-12-02T21:14:40,289 DEBUG [RS:0;87c3fdb6c570:40413 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/87c3fdb6c570:0, corePoolSize=1, maxPoolSize=1 2024-12-02T21:14:40,289 DEBUG [RS:1;87c3fdb6c570:36585 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/87c3fdb6c570:0, corePoolSize=1, maxPoolSize=1 2024-12-02T21:14:40,289 DEBUG [RS:2;87c3fdb6c570:33459 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/87c3fdb6c570:0, corePoolSize=1, maxPoolSize=1 2024-12-02T21:14:40,289 DEBUG [RS:0;87c3fdb6c570:40413 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/87c3fdb6c570:0, corePoolSize=1, maxPoolSize=1 2024-12-02T21:14:40,289 DEBUG [RS:2;87c3fdb6c570:33459 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/87c3fdb6c570:0, corePoolSize=1, maxPoolSize=1 2024-12-02T21:14:40,289 DEBUG [RS:1;87c3fdb6c570:36585 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/87c3fdb6c570:0, corePoolSize=1, maxPoolSize=1 2024-12-02T21:14:40,289 DEBUG [RS:0;87c3fdb6c570:40413 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/87c3fdb6c570:0, corePoolSize=3, maxPoolSize=3 2024-12-02T21:14:40,290 DEBUG [RS:0;87c3fdb6c570:40413 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0, corePoolSize=3, maxPoolSize=3 2024-12-02T21:14:40,290 DEBUG [RS:2;87c3fdb6c570:33459 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/87c3fdb6c570:0, corePoolSize=1, maxPoolSize=1 2024-12-02T21:14:40,290 DEBUG [RS:1;87c3fdb6c570:36585 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/87c3fdb6c570:0, corePoolSize=1, maxPoolSize=1 2024-12-02T21:14:40,290 DEBUG [RS:2;87c3fdb6c570:33459 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/87c3fdb6c570:0, corePoolSize=3, maxPoolSize=3 2024-12-02T21:14:40,290 DEBUG [RS:1;87c3fdb6c570:36585 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/87c3fdb6c570:0, corePoolSize=3, maxPoolSize=3 2024-12-02T21:14:40,290 DEBUG [RS:2;87c3fdb6c570:33459 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0, corePoolSize=3, maxPoolSize=3 2024-12-02T21:14:40,290 DEBUG [RS:1;87c3fdb6c570:36585 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/87c3fdb6c570:0, corePoolSize=3, maxPoolSize=3 2024-12-02T21:14:40,292 INFO [RS:0;87c3fdb6c570:40413 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-02T21:14:40,292 INFO [RS:2;87c3fdb6c570:33459 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-02T21:14:40,293 INFO [RS:1;87c3fdb6c570:36585 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-02T21:14:40,293 INFO [RS:2;87c3fdb6c570:33459 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-02T21:14:40,293 INFO [RS:0;87c3fdb6c570:40413 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-02T21:14:40,293 INFO [RS:1;87c3fdb6c570:36585 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-02T21:14:40,293 INFO [RS:2;87c3fdb6c570:33459 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T21:14:40,293 INFO [RS:0;87c3fdb6c570:40413 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T21:14:40,293 INFO [RS:1;87c3fdb6c570:36585 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T21:14:40,293 INFO [RS:2;87c3fdb6c570:33459 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-02T21:14:40,293 INFO [RS:0;87c3fdb6c570:40413 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-02T21:14:40,293 INFO [RS:1;87c3fdb6c570:36585 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-02T21:14:40,293 INFO [RS:2;87c3fdb6c570:33459 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-02T21:14:40,293 INFO [RS:0;87c3fdb6c570:40413 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-02T21:14:40,293 INFO [RS:1;87c3fdb6c570:36585 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-02T21:14:40,293 INFO [RS:2;87c3fdb6c570:33459 {}] hbase.ChoreService(168): Chore ScheduledChore name=87c3fdb6c570,33459,1733174078298-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-02T21:14:40,293 INFO [RS:0;87c3fdb6c570:40413 {}] hbase.ChoreService(168): Chore ScheduledChore name=87c3fdb6c570,40413,1733174078061-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-02T21:14:40,293 INFO [RS:1;87c3fdb6c570:36585 {}] hbase.ChoreService(168): Chore ScheduledChore name=87c3fdb6c570,36585,1733174078182-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-02T21:14:40,316 INFO [RS:1;87c3fdb6c570:36585 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-02T21:14:40,317 INFO [RS:0;87c3fdb6c570:40413 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-02T21:14:40,317 INFO [RS:2;87c3fdb6c570:33459 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-02T21:14:40,318 INFO [RS:2;87c3fdb6c570:33459 {}] hbase.ChoreService(168): Chore ScheduledChore name=87c3fdb6c570,33459,1733174078298-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T21:14:40,318 INFO [RS:1;87c3fdb6c570:36585 {}] hbase.ChoreService(168): Chore ScheduledChore name=87c3fdb6c570,36585,1733174078182-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T21:14:40,318 INFO [RS:0;87c3fdb6c570:40413 {}] hbase.ChoreService(168): Chore ScheduledChore name=87c3fdb6c570,40413,1733174078061-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T21:14:40,319 INFO [RS:0;87c3fdb6c570:40413 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T21:14:40,319 INFO [RS:2;87c3fdb6c570:33459 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T21:14:40,319 INFO [RS:0;87c3fdb6c570:40413 {}] regionserver.Replication(171): 87c3fdb6c570,40413,1733174078061 started 2024-12-02T21:14:40,319 INFO [RS:2;87c3fdb6c570:33459 {}] regionserver.Replication(171): 87c3fdb6c570,33459,1733174078298 started 2024-12-02T21:14:40,320 INFO [RS:1;87c3fdb6c570:36585 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T21:14:40,320 INFO [RS:1;87c3fdb6c570:36585 {}] regionserver.Replication(171): 87c3fdb6c570,36585,1733174078182 started 2024-12-02T21:14:40,342 INFO [RS:2;87c3fdb6c570:33459 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T21:14:40,342 INFO [RS:1;87c3fdb6c570:36585 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T21:14:40,342 INFO [RS:0;87c3fdb6c570:40413 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T21:14:40,343 INFO [RS:1;87c3fdb6c570:36585 {}] regionserver.HRegionServer(1482): Serving as 87c3fdb6c570,36585,1733174078182, RpcServer on 87c3fdb6c570/172.17.0.3:36585, sessionid=0x10197e5a0060002 2024-12-02T21:14:40,343 INFO [RS:0;87c3fdb6c570:40413 {}] regionserver.HRegionServer(1482): Serving as 87c3fdb6c570,40413,1733174078061, RpcServer on 87c3fdb6c570/172.17.0.3:40413, sessionid=0x10197e5a0060001 2024-12-02T21:14:40,343 INFO [RS:2;87c3fdb6c570:33459 {}] regionserver.HRegionServer(1482): Serving as 87c3fdb6c570,33459,1733174078298, RpcServer on 87c3fdb6c570/172.17.0.3:33459, sessionid=0x10197e5a0060003 2024-12-02T21:14:40,344 DEBUG [RS:0;87c3fdb6c570:40413 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-02T21:14:40,344 DEBUG [RS:2;87c3fdb6c570:33459 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-02T21:14:40,344 DEBUG [RS:1;87c3fdb6c570:36585 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-02T21:14:40,344 DEBUG [RS:2;87c3fdb6c570:33459 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 87c3fdb6c570,33459,1733174078298 2024-12-02T21:14:40,344 DEBUG [RS:0;87c3fdb6c570:40413 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:40,344 DEBUG [RS:1;87c3fdb6c570:36585 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:40,344 DEBUG [RS:2;87c3fdb6c570:33459 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '87c3fdb6c570,33459,1733174078298' 2024-12-02T21:14:40,344 DEBUG [RS:0;87c3fdb6c570:40413 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '87c3fdb6c570,40413,1733174078061' 2024-12-02T21:14:40,344 DEBUG [RS:1;87c3fdb6c570:36585 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '87c3fdb6c570,36585,1733174078182' 2024-12-02T21:14:40,345 DEBUG [RS:2;87c3fdb6c570:33459 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-02T21:14:40,345 DEBUG [RS:0;87c3fdb6c570:40413 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-02T21:14:40,345 DEBUG [RS:1;87c3fdb6c570:36585 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-02T21:14:40,346 DEBUG [RS:2;87c3fdb6c570:33459 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-02T21:14:40,346 DEBUG [RS:1;87c3fdb6c570:36585 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-02T21:14:40,346 DEBUG [RS:0;87c3fdb6c570:40413 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-02T21:14:40,347 DEBUG [RS:1;87c3fdb6c570:36585 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-02T21:14:40,347 DEBUG [RS:1;87c3fdb6c570:36585 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-02T21:14:40,347 DEBUG [RS:0;87c3fdb6c570:40413 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-02T21:14:40,347 DEBUG [RS:0;87c3fdb6c570:40413 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-02T21:14:40,347 DEBUG [RS:1;87c3fdb6c570:36585 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:40,347 DEBUG [RS:0;87c3fdb6c570:40413 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:40,347 DEBUG [RS:1;87c3fdb6c570:36585 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '87c3fdb6c570,36585,1733174078182' 2024-12-02T21:14:40,347 DEBUG [RS:0;87c3fdb6c570:40413 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '87c3fdb6c570,40413,1733174078061' 2024-12-02T21:14:40,347 WARN [87c3fdb6c570:42045 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-02T21:14:40,347 DEBUG [RS:1;87c3fdb6c570:36585 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-02T21:14:40,347 DEBUG [RS:0;87c3fdb6c570:40413 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-02T21:14:40,347 DEBUG [RS:2;87c3fdb6c570:33459 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-02T21:14:40,347 DEBUG [RS:2;87c3fdb6c570:33459 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-02T21:14:40,347 DEBUG [RS:2;87c3fdb6c570:33459 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 87c3fdb6c570,33459,1733174078298 2024-12-02T21:14:40,347 DEBUG [RS:2;87c3fdb6c570:33459 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '87c3fdb6c570,33459,1733174078298' 2024-12-02T21:14:40,347 DEBUG [RS:2;87c3fdb6c570:33459 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-02T21:14:40,348 DEBUG [RS:1;87c3fdb6c570:36585 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-02T21:14:40,348 DEBUG [RS:2;87c3fdb6c570:33459 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-02T21:14:40,348 DEBUG [RS:0;87c3fdb6c570:40413 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-02T21:14:40,348 DEBUG [RS:1;87c3fdb6c570:36585 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-02T21:14:40,348 INFO [RS:1;87c3fdb6c570:36585 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-02T21:14:40,348 INFO [RS:1;87c3fdb6c570:36585 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-02T21:14:40,349 DEBUG [RS:2;87c3fdb6c570:33459 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-02T21:14:40,349 INFO [RS:2;87c3fdb6c570:33459 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-02T21:14:40,349 INFO [RS:2;87c3fdb6c570:33459 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-02T21:14:40,351 DEBUG [RS:0;87c3fdb6c570:40413 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-02T21:14:40,351 INFO [RS:0;87c3fdb6c570:40413 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-02T21:14:40,351 INFO [RS:0;87c3fdb6c570:40413 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-02T21:14:40,459 INFO [RS:1;87c3fdb6c570:36585 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-02T21:14:40,459 INFO [RS:2;87c3fdb6c570:33459 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-02T21:14:40,459 INFO [RS:0;87c3fdb6c570:40413 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-02T21:14:40,462 INFO [RS:2;87c3fdb6c570:33459 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=87c3fdb6c570%2C33459%2C1733174078298, suffix=, logDir=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/WALs/87c3fdb6c570,33459,1733174078298, archiveDir=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/oldWALs, maxLogs=32 2024-12-02T21:14:40,462 INFO [RS:0;87c3fdb6c570:40413 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=87c3fdb6c570%2C40413%2C1733174078061, suffix=, logDir=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/WALs/87c3fdb6c570,40413,1733174078061, archiveDir=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/oldWALs, maxLogs=32 2024-12-02T21:14:40,462 INFO [RS:1;87c3fdb6c570:36585 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=87c3fdb6c570%2C36585%2C1733174078182, suffix=, logDir=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/WALs/87c3fdb6c570,36585,1733174078182, archiveDir=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/oldWALs, maxLogs=32 2024-12-02T21:14:40,479 DEBUG [RS:1;87c3fdb6c570:36585 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/WALs/87c3fdb6c570,36585,1733174078182/87c3fdb6c570%2C36585%2C1733174078182.1733174080466, exclude list is [], retry=0 2024-12-02T21:14:40,479 DEBUG [RS:2;87c3fdb6c570:33459 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/WALs/87c3fdb6c570,33459,1733174078298/87c3fdb6c570%2C33459%2C1733174078298.1733174080466, exclude list is [], retry=0 2024-12-02T21:14:40,479 DEBUG [RS:0;87c3fdb6c570:40413 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/WALs/87c3fdb6c570,40413,1733174078061/87c3fdb6c570%2C40413%2C1733174078061.1733174080466, exclude list is [], retry=0 2024-12-02T21:14:40,484 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36203,DS-2d489d52-306c-448f-8298-290ac4052947,DISK] 2024-12-02T21:14:40,485 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40959,DS-54bbf138-7100-4d62-a6b6-08df6ef7aa28,DISK] 2024-12-02T21:14:40,485 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36203,DS-2d489d52-306c-448f-8298-290ac4052947,DISK] 2024-12-02T21:14:40,485 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42827,DS-9aa11248-a7e5-4660-b440-2c286bf86b1b,DISK] 2024-12-02T21:14:40,485 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40959,DS-54bbf138-7100-4d62-a6b6-08df6ef7aa28,DISK] 2024-12-02T21:14:40,485 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42827,DS-9aa11248-a7e5-4660-b440-2c286bf86b1b,DISK] 2024-12-02T21:14:40,486 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36203,DS-2d489d52-306c-448f-8298-290ac4052947,DISK] 2024-12-02T21:14:40,486 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40959,DS-54bbf138-7100-4d62-a6b6-08df6ef7aa28,DISK] 2024-12-02T21:14:40,487 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42827,DS-9aa11248-a7e5-4660-b440-2c286bf86b1b,DISK] 2024-12-02T21:14:40,503 INFO [RS:2;87c3fdb6c570:33459 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/WALs/87c3fdb6c570,33459,1733174078298/87c3fdb6c570%2C33459%2C1733174078298.1733174080466 2024-12-02T21:14:40,504 INFO [RS:1;87c3fdb6c570:36585 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/WALs/87c3fdb6c570,36585,1733174078182/87c3fdb6c570%2C36585%2C1733174078182.1733174080466 2024-12-02T21:14:40,504 INFO [RS:0;87c3fdb6c570:40413 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/WALs/87c3fdb6c570,40413,1733174078061/87c3fdb6c570%2C40413%2C1733174078061.1733174080466 2024-12-02T21:14:40,505 DEBUG [RS:2;87c3fdb6c570:33459 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46475:46475),(127.0.0.1/127.0.0.1:35813:35813),(127.0.0.1/127.0.0.1:34107:34107)] 2024-12-02T21:14:40,505 DEBUG [RS:1;87c3fdb6c570:36585 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46475:46475),(127.0.0.1/127.0.0.1:35813:35813),(127.0.0.1/127.0.0.1:34107:34107)] 2024-12-02T21:14:40,506 DEBUG [RS:0;87c3fdb6c570:40413 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46475:46475),(127.0.0.1/127.0.0.1:34107:34107),(127.0.0.1/127.0.0.1:35813:35813)] 2024-12-02T21:14:40,599 DEBUG [87c3fdb6c570:42045 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-12-02T21:14:40,608 DEBUG [87c3fdb6c570:42045 {}] balancer.BalancerClusterState(204): Hosts are {87c3fdb6c570=0} racks are {/default-rack=0} 2024-12-02T21:14:40,615 DEBUG [87c3fdb6c570:42045 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-02T21:14:40,615 DEBUG [87c3fdb6c570:42045 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-02T21:14:40,615 DEBUG [87c3fdb6c570:42045 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-02T21:14:40,615 DEBUG [87c3fdb6c570:42045 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-02T21:14:40,615 DEBUG [87c3fdb6c570:42045 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-02T21:14:40,615 DEBUG [87c3fdb6c570:42045 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-02T21:14:40,615 INFO [87c3fdb6c570:42045 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-02T21:14:40,615 INFO [87c3fdb6c570:42045 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-02T21:14:40,615 INFO [87c3fdb6c570:42045 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-02T21:14:40,615 DEBUG [87c3fdb6c570:42045 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-02T21:14:40,622 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=87c3fdb6c570,33459,1733174078298 2024-12-02T21:14:40,628 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 87c3fdb6c570,33459,1733174078298, state=OPENING 2024-12-02T21:14:40,645 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-02T21:14:40,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36585-0x10197e5a0060002, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T21:14:40,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40413-0x10197e5a0060001, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T21:14:40,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33459-0x10197e5a0060003, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T21:14:40,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42045-0x10197e5a0060000, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T21:14:40,657 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:40,657 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:40,657 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:40,657 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:40,659 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T21:14:40,662 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=87c3fdb6c570,33459,1733174078298}] 2024-12-02T21:14:40,843 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-02T21:14:40,845 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57051, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-02T21:14:40,857 INFO [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-02T21:14:40,857 INFO [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T21:14:40,858 INFO [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-02T21:14:40,861 INFO [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=87c3fdb6c570%2C33459%2C1733174078298.meta, suffix=.meta, logDir=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/WALs/87c3fdb6c570,33459,1733174078298, archiveDir=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/oldWALs, maxLogs=32 2024-12-02T21:14:40,875 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/WALs/87c3fdb6c570,33459,1733174078298/87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, exclude list is [], retry=0 2024-12-02T21:14:40,879 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40959,DS-54bbf138-7100-4d62-a6b6-08df6ef7aa28,DISK] 2024-12-02T21:14:40,879 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36203,DS-2d489d52-306c-448f-8298-290ac4052947,DISK] 2024-12-02T21:14:40,880 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42827,DS-9aa11248-a7e5-4660-b440-2c286bf86b1b,DISK] 2024-12-02T21:14:40,885 INFO [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/WALs/87c3fdb6c570,33459,1733174078298/87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta 2024-12-02T21:14:40,885 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35813:35813),(127.0.0.1/127.0.0.1:46475:46475),(127.0.0.1/127.0.0.1:34107:34107)] 2024-12-02T21:14:40,886 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-02T21:14:40,888 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-02T21:14:40,891 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-02T21:14:40,896 INFO [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-02T21:14:40,901 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-02T21:14:40,901 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:14:40,902 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-02T21:14:40,902 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-02T21:14:40,905 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-02T21:14:40,907 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-02T21:14:40,907 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:14:40,908 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T21:14:40,908 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-02T21:14:40,910 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-02T21:14:40,910 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:14:40,911 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T21:14:40,912 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-02T21:14:40,913 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-02T21:14:40,913 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:14:40,914 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T21:14:40,915 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-02T21:14:40,917 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-02T21:14:40,917 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:14:40,918 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T21:14:40,918 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-02T21:14:40,920 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/hbase/meta/1588230740 2024-12-02T21:14:40,923 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/hbase/meta/1588230740 2024-12-02T21:14:40,926 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-02T21:14:40,926 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-02T21:14:40,927 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-02T21:14:40,930 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-02T21:14:40,932 INFO [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71378710, jitterRate=0.06362566351890564}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-02T21:14:40,932 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-02T21:14:40,934 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733174080902Writing region info on filesystem at 1733174080903 (+1 ms)Initializing all the Stores at 1733174080905 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733174080905Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733174080905Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174080905Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733174080905Cleaning up temporary data from old regions at 1733174080926 (+21 ms)Running coprocessor post-open hooks at 1733174080932 (+6 ms)Region opened successfully at 1733174080934 (+2 ms) 2024-12-02T21:14:40,940 INFO [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733174080835 2024-12-02T21:14:40,950 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-02T21:14:40,950 INFO [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-02T21:14:40,952 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=87c3fdb6c570,33459,1733174078298 2024-12-02T21:14:40,954 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 87c3fdb6c570,33459,1733174078298, state=OPEN 2024-12-02T21:14:41,013 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33459-0x10197e5a0060003, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:41,013 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36585-0x10197e5a0060002, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:41,013 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40413-0x10197e5a0060001, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:41,013 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42045-0x10197e5a0060000, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:41,014 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:41,014 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:41,014 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:41,014 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:41,014 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=87c3fdb6c570,33459,1733174078298 2024-12-02T21:14:41,019 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-02T21:14:41,019 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=87c3fdb6c570,33459,1733174078298 in 352 msec 2024-12-02T21:14:41,028 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-02T21:14:41,028 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 840 msec 2024-12-02T21:14:41,030 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-02T21:14:41,030 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-02T21:14:41,049 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:14:41,050 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,33459,1733174078298, seqNum=-1] 2024-12-02T21:14:41,068 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:14:41,070 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58037, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:14:41,089 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1270 sec 2024-12-02T21:14:41,089 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733174081089, completionTime=-1 2024-12-02T21:14:41,092 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-12-02T21:14:41,092 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-02T21:14:41,119 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-12-02T21:14:41,119 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733174141119 2024-12-02T21:14:41,119 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733174201119 2024-12-02T21:14:41,119 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 26 msec 2024-12-02T21:14:41,120 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-02T21:14:41,126 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=87c3fdb6c570,42045,1733174077295-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T21:14:41,127 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=87c3fdb6c570,42045,1733174077295-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T21:14:41,127 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=87c3fdb6c570,42045,1733174077295-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T21:14:41,128 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-87c3fdb6c570:42045, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T21:14:41,128 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-02T21:14:41,129 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-02T21:14:41,135 DEBUG [master/87c3fdb6c570:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-02T21:14:41,157 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.683sec 2024-12-02T21:14:41,158 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-02T21:14:41,159 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-02T21:14:41,160 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-02T21:14:41,161 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-02T21:14:41,161 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-02T21:14:41,162 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=87c3fdb6c570,42045,1733174077295-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-02T21:14:41,162 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=87c3fdb6c570,42045,1733174077295-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-02T21:14:41,166 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-02T21:14:41,167 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-02T21:14:41,167 INFO [master/87c3fdb6c570:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=87c3fdb6c570,42045,1733174077295-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T21:14:41,194 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fac59c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:14:41,198 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-02T21:14:41,198 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-02T21:14:41,202 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 87c3fdb6c570,42045,-1 for getting cluster id 2024-12-02T21:14:41,205 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T21:14:41,214 DEBUG [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a3fb6e34-ca27-4453-9e72-69b912f26690' 2024-12-02T21:14:41,216 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T21:14:41,216 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a3fb6e34-ca27-4453-9e72-69b912f26690" 2024-12-02T21:14:41,218 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@713d8da4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:14:41,218 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [87c3fdb6c570,42045,-1] 2024-12-02T21:14:41,221 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T21:14:41,223 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:14:41,223 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50808, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T21:14:41,226 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3edc677d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T21:14:41,227 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:14:41,233 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,33459,1733174078298, seqNum=-1] 2024-12-02T21:14:41,233 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:14:41,236 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56190, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:14:41,256 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=87c3fdb6c570,42045,1733174077295 2024-12-02T21:14:41,276 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-02T21:14:41,282 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 87c3fdb6c570,42045,1733174077295 2024-12-02T21:14:41,285 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5e7ce942 2024-12-02T21:14:41,286 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-02T21:14:41,289 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50824, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-02T21:14:41,297 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42045 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'test', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T21:14:41,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42045 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test 2024-12-02T21:14:41,314 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_PRE_OPERATION 2024-12-02T21:14:41,316 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42045 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "test" procId is: 4 2024-12-02T21:14:41,317 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:14:41,320 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-02T21:14:41,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T21:14:41,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741837_1013 (size=358) 2024-12-02T21:14:41,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36203 is added to blk_1073741837_1013 (size=358) 2024-12-02T21:14:41,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42827 is added to blk_1073741837_1013 (size=358) 2024-12-02T21:14:41,337 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 85ca3a9a09d75c0941c58f5b83584a8e, NAME => 'test,,1733174081291.85ca3a9a09d75c0941c58f5b83584a8e.', STARTKEY => '', ENDKEY => 'bbb'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b 2024-12-02T21:14:41,337 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 0517912ff18e81317bb5474cefaca025, NAME => 'test,ccc,1733174081291.0517912ff18e81317bb5474cefaca025.', STARTKEY => 'ccc', ENDKEY => 'ddd'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b 2024-12-02T21:14:41,338 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 5ed86f04f519cc5dccd1b6d41be3da7d, NAME => 'test,bbb,1733174081291.5ed86f04f519cc5dccd1b6d41be3da7d.', STARTKEY => 'bbb', ENDKEY => 'ccc'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b 2024-12-02T21:14:41,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741839_1015 (size=45) 2024-12-02T21:14:41,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36203 is added to blk_1073741839_1015 (size=45) 2024-12-02T21:14:41,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42827 is added to blk_1073741839_1015 (size=45) 2024-12-02T21:14:41,360 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,bbb,1733174081291.5ed86f04f519cc5dccd1b6d41be3da7d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:14:41,360 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 5ed86f04f519cc5dccd1b6d41be3da7d, disabling compactions & flushes 2024-12-02T21:14:41,360 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,bbb,1733174081291.5ed86f04f519cc5dccd1b6d41be3da7d. 2024-12-02T21:14:41,360 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1733174081291.5ed86f04f519cc5dccd1b6d41be3da7d. 2024-12-02T21:14:41,360 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1733174081291.5ed86f04f519cc5dccd1b6d41be3da7d. after waiting 0 ms 2024-12-02T21:14:41,360 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1733174081291.5ed86f04f519cc5dccd1b6d41be3da7d. 2024-12-02T21:14:41,360 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,bbb,1733174081291.5ed86f04f519cc5dccd1b6d41be3da7d. 2024-12-02T21:14:41,360 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 5ed86f04f519cc5dccd1b6d41be3da7d: Waiting for close lock at 1733174081360Disabling compacts and flushes for region at 1733174081360Disabling writes for close at 1733174081360Writing region close event to WAL at 1733174081360Closed at 1733174081360 2024-12-02T21:14:41,361 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => ff611163b4ab91c50d940651a0065c34, NAME => 'test,ddd,1733174081291.ff611163b4ab91c50d940651a0065c34.', STARTKEY => 'ddd', ENDKEY => 'eee'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b 2024-12-02T21:14:41,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36203 is added to blk_1073741838_1014 (size=45) 2024-12-02T21:14:41,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42827 is added to blk_1073741838_1014 (size=45) 2024-12-02T21:14:41,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741838_1014 (size=45) 2024-12-02T21:14:41,369 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ccc,1733174081291.0517912ff18e81317bb5474cefaca025.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:14:41,370 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 0517912ff18e81317bb5474cefaca025, disabling compactions & flushes 2024-12-02T21:14:41,370 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ccc,1733174081291.0517912ff18e81317bb5474cefaca025. 2024-12-02T21:14:41,370 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1733174081291.0517912ff18e81317bb5474cefaca025. 2024-12-02T21:14:41,370 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1733174081291.0517912ff18e81317bb5474cefaca025. after waiting 0 ms 2024-12-02T21:14:41,370 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1733174081291.0517912ff18e81317bb5474cefaca025. 2024-12-02T21:14:41,370 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ccc,1733174081291.0517912ff18e81317bb5474cefaca025. 2024-12-02T21:14:41,370 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 0517912ff18e81317bb5474cefaca025: Waiting for close lock at 1733174081369Disabling compacts and flushes for region at 1733174081369Disabling writes for close at 1733174081370 (+1 ms)Writing region close event to WAL at 1733174081370Closed at 1733174081370 2024-12-02T21:14:41,370 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 120cf3776e92a15d3276f11911c4042f, NAME => 'test,eee,1733174081291.120cf3776e92a15d3276f11911c4042f.', STARTKEY => 'eee', ENDKEY => 'fff'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b 2024-12-02T21:14:41,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36203 is added to blk_1073741840_1016 (size=42) 2024-12-02T21:14:41,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42827 is added to blk_1073741840_1016 (size=42) 2024-12-02T21:14:41,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741840_1016 (size=42) 2024-12-02T21:14:41,379 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,,1733174081291.85ca3a9a09d75c0941c58f5b83584a8e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:14:41,380 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 85ca3a9a09d75c0941c58f5b83584a8e, disabling compactions & flushes 2024-12-02T21:14:41,380 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,,1733174081291.85ca3a9a09d75c0941c58f5b83584a8e. 2024-12-02T21:14:41,380 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1733174081291.85ca3a9a09d75c0941c58f5b83584a8e. 2024-12-02T21:14:41,380 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,,1733174081291.85ca3a9a09d75c0941c58f5b83584a8e. after waiting 0 ms 2024-12-02T21:14:41,380 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,,1733174081291.85ca3a9a09d75c0941c58f5b83584a8e. 2024-12-02T21:14:41,380 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,,1733174081291.85ca3a9a09d75c0941c58f5b83584a8e. 2024-12-02T21:14:41,380 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 85ca3a9a09d75c0941c58f5b83584a8e: Waiting for close lock at 1733174081380Disabling compacts and flushes for region at 1733174081380Disabling writes for close at 1733174081380Writing region close event to WAL at 1733174081380Closed at 1733174081380 2024-12-02T21:14:41,381 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e87d99b937228303e28b7ea0373ddc70, NAME => 'test,fff,1733174081291.e87d99b937228303e28b7ea0373ddc70.', STARTKEY => 'fff', ENDKEY => 'ggg'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b 2024-12-02T21:14:41,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741842_1018 (size=45) 2024-12-02T21:14:41,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42827 is added to blk_1073741841_1017 (size=45) 2024-12-02T21:14:41,391 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ddd,1733174081291.ff611163b4ab91c50d940651a0065c34.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:14:41,392 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing ff611163b4ab91c50d940651a0065c34, disabling compactions & flushes 2024-12-02T21:14:41,392 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,eee,1733174081291.120cf3776e92a15d3276f11911c4042f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:14:41,392 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ddd,1733174081291.ff611163b4ab91c50d940651a0065c34. 2024-12-02T21:14:41,392 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1733174081291.ff611163b4ab91c50d940651a0065c34. 2024-12-02T21:14:41,392 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 120cf3776e92a15d3276f11911c4042f, disabling compactions & flushes 2024-12-02T21:14:41,392 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1733174081291.ff611163b4ab91c50d940651a0065c34. after waiting 0 ms 2024-12-02T21:14:41,392 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1733174081291.ff611163b4ab91c50d940651a0065c34. 2024-12-02T21:14:41,392 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,eee,1733174081291.120cf3776e92a15d3276f11911c4042f. 2024-12-02T21:14:41,392 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ddd,1733174081291.ff611163b4ab91c50d940651a0065c34. 2024-12-02T21:14:41,392 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1733174081291.120cf3776e92a15d3276f11911c4042f. 2024-12-02T21:14:41,392 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for ff611163b4ab91c50d940651a0065c34: Waiting for close lock at 1733174081391Disabling compacts and flushes for region at 1733174081391Disabling writes for close at 1733174081392 (+1 ms)Writing region close event to WAL at 1733174081392Closed at 1733174081392 2024-12-02T21:14:41,392 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,eee,1733174081291.120cf3776e92a15d3276f11911c4042f. after waiting 0 ms 2024-12-02T21:14:41,392 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,eee,1733174081291.120cf3776e92a15d3276f11911c4042f. 2024-12-02T21:14:41,392 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,eee,1733174081291.120cf3776e92a15d3276f11911c4042f. 2024-12-02T21:14:41,392 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 120cf3776e92a15d3276f11911c4042f: Waiting for close lock at 1733174081392Disabling compacts and flushes for region at 1733174081392Disabling writes for close at 1733174081392Writing region close event to WAL at 1733174081392Closed at 1733174081392 2024-12-02T21:14:41,393 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 0dfd4169e4c254c51bcc70a472f99ecf, NAME => 'test,ggg,1733174081291.0dfd4169e4c254c51bcc70a472f99ecf.', STARTKEY => 'ggg', ENDKEY => 'hhh'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b 2024-12-02T21:14:41,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741841_1017 (size=45) 2024-12-02T21:14:41,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42827 is added to blk_1073741842_1018 (size=45) 2024-12-02T21:14:41,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36203 is added to blk_1073741842_1018 (size=45) 2024-12-02T21:14:41,394 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 9950441eda7548fa66872811a13f5bf0, NAME => 'test,hhh,1733174081291.9950441eda7548fa66872811a13f5bf0.', STARTKEY => 'hhh', ENDKEY => 'iii'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b 2024-12-02T21:14:41,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36203 is added to blk_1073741841_1017 (size=45) 2024-12-02T21:14:41,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741843_1019 (size=45) 2024-12-02T21:14:41,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42827 is added to blk_1073741843_1019 (size=45) 2024-12-02T21:14:41,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36203 is added to blk_1073741843_1019 (size=45) 2024-12-02T21:14:41,402 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,fff,1733174081291.e87d99b937228303e28b7ea0373ddc70.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:14:41,402 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing e87d99b937228303e28b7ea0373ddc70, disabling compactions & flushes 2024-12-02T21:14:41,402 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,fff,1733174081291.e87d99b937228303e28b7ea0373ddc70. 2024-12-02T21:14:41,402 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1733174081291.e87d99b937228303e28b7ea0373ddc70. 2024-12-02T21:14:41,402 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,fff,1733174081291.e87d99b937228303e28b7ea0373ddc70. after waiting 0 ms 2024-12-02T21:14:41,402 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,fff,1733174081291.e87d99b937228303e28b7ea0373ddc70. 2024-12-02T21:14:41,402 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,fff,1733174081291.e87d99b937228303e28b7ea0373ddc70. 2024-12-02T21:14:41,402 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for e87d99b937228303e28b7ea0373ddc70: Waiting for close lock at 1733174081402Disabling compacts and flushes for region at 1733174081402Disabling writes for close at 1733174081402Writing region close event to WAL at 1733174081402Closed at 1733174081402 2024-12-02T21:14:41,403 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8e039115fcc0d0a54e2d03efed7e05de, NAME => 'test,iii,1733174081291.8e039115fcc0d0a54e2d03efed7e05de.', STARTKEY => 'iii', ENDKEY => 'jjj'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b 2024-12-02T21:14:41,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36203 is added to blk_1073741845_1021 (size=45) 2024-12-02T21:14:41,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741845_1021 (size=45) 2024-12-02T21:14:41,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42827 is added to blk_1073741845_1021 (size=45) 2024-12-02T21:14:41,411 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,hhh,1733174081291.9950441eda7548fa66872811a13f5bf0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:14:41,411 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 9950441eda7548fa66872811a13f5bf0, disabling compactions & flushes 2024-12-02T21:14:41,411 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,hhh,1733174081291.9950441eda7548fa66872811a13f5bf0. 2024-12-02T21:14:41,411 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1733174081291.9950441eda7548fa66872811a13f5bf0. 2024-12-02T21:14:41,411 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1733174081291.9950441eda7548fa66872811a13f5bf0. after waiting 0 ms 2024-12-02T21:14:41,411 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1733174081291.9950441eda7548fa66872811a13f5bf0. 2024-12-02T21:14:41,411 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,hhh,1733174081291.9950441eda7548fa66872811a13f5bf0. 2024-12-02T21:14:41,411 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 9950441eda7548fa66872811a13f5bf0: Waiting for close lock at 1733174081411Disabling compacts and flushes for region at 1733174081411Disabling writes for close at 1733174081411Writing region close event to WAL at 1733174081411Closed at 1733174081411 2024-12-02T21:14:41,412 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 5d60c566b39516dd875aaf2bd7def402, NAME => 'test,jjj,1733174081291.5d60c566b39516dd875aaf2bd7def402.', STARTKEY => 'jjj', ENDKEY => 'kkk'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b 2024-12-02T21:14:41,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42827 is added to blk_1073741844_1020 (size=45) 2024-12-02T21:14:41,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36203 is added to blk_1073741844_1020 (size=45) 2024-12-02T21:14:41,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741844_1020 (size=45) 2024-12-02T21:14:41,416 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ggg,1733174081291.0dfd4169e4c254c51bcc70a472f99ecf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:14:41,416 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 0dfd4169e4c254c51bcc70a472f99ecf, disabling compactions & flushes 2024-12-02T21:14:41,416 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ggg,1733174081291.0dfd4169e4c254c51bcc70a472f99ecf. 2024-12-02T21:14:41,417 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1733174081291.0dfd4169e4c254c51bcc70a472f99ecf. 2024-12-02T21:14:41,417 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1733174081291.0dfd4169e4c254c51bcc70a472f99ecf. after waiting 0 ms 2024-12-02T21:14:41,417 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1733174081291.0dfd4169e4c254c51bcc70a472f99ecf. 2024-12-02T21:14:41,417 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ggg,1733174081291.0dfd4169e4c254c51bcc70a472f99ecf. 2024-12-02T21:14:41,417 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 0dfd4169e4c254c51bcc70a472f99ecf: Waiting for close lock at 1733174081416Disabling compacts and flushes for region at 1733174081416Disabling writes for close at 1733174081417 (+1 ms)Writing region close event to WAL at 1733174081417Closed at 1733174081417 2024-12-02T21:14:41,418 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => a524d88be9ac29b416f0a6c04627fa1c, NAME => 'test,kkk,1733174081291.a524d88be9ac29b416f0a6c04627fa1c.', STARTKEY => 'kkk', ENDKEY => 'lll'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b 2024-12-02T21:14:41,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36203 is added to blk_1073741846_1022 (size=45) 2024-12-02T21:14:41,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741846_1022 (size=45) 2024-12-02T21:14:41,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T21:14:41,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42827 is added to blk_1073741846_1022 (size=45) 2024-12-02T21:14:41,433 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,iii,1733174081291.8e039115fcc0d0a54e2d03efed7e05de.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:14:41,433 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 8e039115fcc0d0a54e2d03efed7e05de, disabling compactions & flushes 2024-12-02T21:14:41,434 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,iii,1733174081291.8e039115fcc0d0a54e2d03efed7e05de. 2024-12-02T21:14:41,434 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1733174081291.8e039115fcc0d0a54e2d03efed7e05de. 2024-12-02T21:14:41,434 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,iii,1733174081291.8e039115fcc0d0a54e2d03efed7e05de. after waiting 0 ms 2024-12-02T21:14:41,434 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,iii,1733174081291.8e039115fcc0d0a54e2d03efed7e05de. 2024-12-02T21:14:41,434 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,iii,1733174081291.8e039115fcc0d0a54e2d03efed7e05de. 2024-12-02T21:14:41,434 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8e039115fcc0d0a54e2d03efed7e05de: Waiting for close lock at 1733174081433Disabling compacts and flushes for region at 1733174081433Disabling writes for close at 1733174081434 (+1 ms)Writing region close event to WAL at 1733174081434Closed at 1733174081434 2024-12-02T21:14:41,435 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0d0be65f6ffd7ebb4c8b2fee37042136, NAME => 'test,lll,1733174081291.0d0be65f6ffd7ebb4c8b2fee37042136.', STARTKEY => 'lll', ENDKEY => 'mmm'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b 2024-12-02T21:14:41,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42827 is added to blk_1073741847_1023 (size=45) 2024-12-02T21:14:41,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741847_1023 (size=45) 2024-12-02T21:14:41,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36203 is added to blk_1073741847_1023 (size=45) 2024-12-02T21:14:41,446 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,kkk,1733174081291.a524d88be9ac29b416f0a6c04627fa1c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:14:41,446 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing a524d88be9ac29b416f0a6c04627fa1c, disabling compactions & flushes 2024-12-02T21:14:41,446 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,kkk,1733174081291.a524d88be9ac29b416f0a6c04627fa1c. 2024-12-02T21:14:41,446 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1733174081291.a524d88be9ac29b416f0a6c04627fa1c. 2024-12-02T21:14:41,446 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1733174081291.a524d88be9ac29b416f0a6c04627fa1c. after waiting 0 ms 2024-12-02T21:14:41,446 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1733174081291.a524d88be9ac29b416f0a6c04627fa1c. 2024-12-02T21:14:41,447 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,kkk,1733174081291.a524d88be9ac29b416f0a6c04627fa1c. 2024-12-02T21:14:41,447 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for a524d88be9ac29b416f0a6c04627fa1c: Waiting for close lock at 1733174081446Disabling compacts and flushes for region at 1733174081446Disabling writes for close at 1733174081446Writing region close event to WAL at 1733174081447 (+1 ms)Closed at 1733174081447 2024-12-02T21:14:41,448 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 292fd0b4e0f07516cc25c069181e3111, NAME => 'test,mmm,1733174081291.292fd0b4e0f07516cc25c069181e3111.', STARTKEY => 'mmm', ENDKEY => 'nnn'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b 2024-12-02T21:14:41,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741848_1024 (size=45) 2024-12-02T21:14:41,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36203 is added to blk_1073741848_1024 (size=45) 2024-12-02T21:14:41,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42827 is added to blk_1073741849_1025 (size=45) 2024-12-02T21:14:41,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741849_1025 (size=45) 2024-12-02T21:14:41,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36203 is added to blk_1073741849_1025 (size=45) 2024-12-02T21:14:41,460 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,lll,1733174081291.0d0be65f6ffd7ebb4c8b2fee37042136.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:14:41,461 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 0d0be65f6ffd7ebb4c8b2fee37042136, disabling compactions & flushes 2024-12-02T21:14:41,461 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,lll,1733174081291.0d0be65f6ffd7ebb4c8b2fee37042136. 2024-12-02T21:14:41,461 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1733174081291.0d0be65f6ffd7ebb4c8b2fee37042136. 2024-12-02T21:14:41,461 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,lll,1733174081291.0d0be65f6ffd7ebb4c8b2fee37042136. after waiting 0 ms 2024-12-02T21:14:41,461 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,lll,1733174081291.0d0be65f6ffd7ebb4c8b2fee37042136. 2024-12-02T21:14:41,461 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,lll,1733174081291.0d0be65f6ffd7ebb4c8b2fee37042136. 2024-12-02T21:14:41,461 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0d0be65f6ffd7ebb4c8b2fee37042136: Waiting for close lock at 1733174081461Disabling compacts and flushes for region at 1733174081461Disabling writes for close at 1733174081461Writing region close event to WAL at 1733174081461Closed at 1733174081461 2024-12-02T21:14:41,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42827 is added to blk_1073741848_1024 (size=45) 2024-12-02T21:14:41,462 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 87ba365a3e5d028c9193155af96aeb60, NAME => 'test,nnn,1733174081291.87ba365a3e5d028c9193155af96aeb60.', STARTKEY => 'nnn', ENDKEY => 'ooo'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b 2024-12-02T21:14:41,462 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,jjj,1733174081291.5d60c566b39516dd875aaf2bd7def402.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:14:41,463 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 5d60c566b39516dd875aaf2bd7def402, disabling compactions & flushes 2024-12-02T21:14:41,463 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,jjj,1733174081291.5d60c566b39516dd875aaf2bd7def402. 2024-12-02T21:14:41,463 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1733174081291.5d60c566b39516dd875aaf2bd7def402. 2024-12-02T21:14:41,463 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1733174081291.5d60c566b39516dd875aaf2bd7def402. after waiting 0 ms 2024-12-02T21:14:41,463 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1733174081291.5d60c566b39516dd875aaf2bd7def402. 2024-12-02T21:14:41,463 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,jjj,1733174081291.5d60c566b39516dd875aaf2bd7def402. 2024-12-02T21:14:41,463 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 5d60c566b39516dd875aaf2bd7def402: Waiting for close lock at 1733174081463Disabling compacts and flushes for region at 1733174081463Disabling writes for close at 1733174081463Writing region close event to WAL at 1733174081463Closed at 1733174081463 2024-12-02T21:14:41,464 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 4197cd4f7ea2f3d6256be6836e9834da, NAME => 'test,ooo,1733174081291.4197cd4f7ea2f3d6256be6836e9834da.', STARTKEY => 'ooo', ENDKEY => 'ppp'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b 2024-12-02T21:14:41,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42827 is added to blk_1073741850_1026 (size=45) 2024-12-02T21:14:41,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36203 is added to blk_1073741850_1026 (size=45) 2024-12-02T21:14:41,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741850_1026 (size=45) 2024-12-02T21:14:41,473 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,mmm,1733174081291.292fd0b4e0f07516cc25c069181e3111.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:14:41,473 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 292fd0b4e0f07516cc25c069181e3111, disabling compactions & flushes 2024-12-02T21:14:41,474 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,mmm,1733174081291.292fd0b4e0f07516cc25c069181e3111. 2024-12-02T21:14:41,474 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1733174081291.292fd0b4e0f07516cc25c069181e3111. 2024-12-02T21:14:41,474 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1733174081291.292fd0b4e0f07516cc25c069181e3111. after waiting 0 ms 2024-12-02T21:14:41,474 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1733174081291.292fd0b4e0f07516cc25c069181e3111. 2024-12-02T21:14:41,474 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,mmm,1733174081291.292fd0b4e0f07516cc25c069181e3111. 2024-12-02T21:14:41,474 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 292fd0b4e0f07516cc25c069181e3111: Waiting for close lock at 1733174081473Disabling compacts and flushes for region at 1733174081473Disabling writes for close at 1733174081474 (+1 ms)Writing region close event to WAL at 1733174081474Closed at 1733174081474 2024-12-02T21:14:41,474 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 20545c8ee2d66091a91ff9697d1f18ca, NAME => 'test,ppp,1733174081291.20545c8ee2d66091a91ff9697d1f18ca.', STARTKEY => 'ppp', ENDKEY => 'qqq'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b 2024-12-02T21:14:41,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42827 is added to blk_1073741851_1027 (size=45) 2024-12-02T21:14:41,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36203 is added to blk_1073741851_1027 (size=45) 2024-12-02T21:14:41,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741851_1027 (size=45) 2024-12-02T21:14:41,481 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,nnn,1733174081291.87ba365a3e5d028c9193155af96aeb60.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:14:41,481 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 87ba365a3e5d028c9193155af96aeb60, disabling compactions & flushes 2024-12-02T21:14:41,481 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,nnn,1733174081291.87ba365a3e5d028c9193155af96aeb60. 2024-12-02T21:14:41,481 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1733174081291.87ba365a3e5d028c9193155af96aeb60. 2024-12-02T21:14:41,481 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1733174081291.87ba365a3e5d028c9193155af96aeb60. after waiting 0 ms 2024-12-02T21:14:41,481 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1733174081291.87ba365a3e5d028c9193155af96aeb60. 2024-12-02T21:14:41,481 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,nnn,1733174081291.87ba365a3e5d028c9193155af96aeb60. 2024-12-02T21:14:41,481 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 87ba365a3e5d028c9193155af96aeb60: Waiting for close lock at 1733174081481Disabling compacts and flushes for region at 1733174081481Disabling writes for close at 1733174081481Writing region close event to WAL at 1733174081481Closed at 1733174081481 2024-12-02T21:14:41,482 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a50bae1d5cb3d03a86dc103b09a0f68b, NAME => 'test,qqq,1733174081291.a50bae1d5cb3d03a86dc103b09a0f68b.', STARTKEY => 'qqq', ENDKEY => 'rrr'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b 2024-12-02T21:14:41,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42827 is added to blk_1073741852_1028 (size=45) 2024-12-02T21:14:41,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36203 is added to blk_1073741852_1028 (size=45) 2024-12-02T21:14:41,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741852_1028 (size=45) 2024-12-02T21:14:41,491 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ooo,1733174081291.4197cd4f7ea2f3d6256be6836e9834da.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:14:41,491 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 4197cd4f7ea2f3d6256be6836e9834da, disabling compactions & flushes 2024-12-02T21:14:41,491 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ooo,1733174081291.4197cd4f7ea2f3d6256be6836e9834da. 2024-12-02T21:14:41,491 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1733174081291.4197cd4f7ea2f3d6256be6836e9834da. 2024-12-02T21:14:41,492 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1733174081291.4197cd4f7ea2f3d6256be6836e9834da. after waiting 0 ms 2024-12-02T21:14:41,492 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1733174081291.4197cd4f7ea2f3d6256be6836e9834da. 2024-12-02T21:14:41,492 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ooo,1733174081291.4197cd4f7ea2f3d6256be6836e9834da. 2024-12-02T21:14:41,492 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 4197cd4f7ea2f3d6256be6836e9834da: Waiting for close lock at 1733174081491Disabling compacts and flushes for region at 1733174081491Disabling writes for close at 1733174081492 (+1 ms)Writing region close event to WAL at 1733174081492Closed at 1733174081492 2024-12-02T21:14:41,493 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 215753e705337ae11b1b38ad77b6a5cd, NAME => 'test,rrr,1733174081291.215753e705337ae11b1b38ad77b6a5cd.', STARTKEY => 'rrr', ENDKEY => 'sss'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b 2024-12-02T21:14:41,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42827 is added to blk_1073741853_1029 (size=45) 2024-12-02T21:14:41,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36203 is added to blk_1073741853_1029 (size=45) 2024-12-02T21:14:41,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741853_1029 (size=45) 2024-12-02T21:14:41,497 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ppp,1733174081291.20545c8ee2d66091a91ff9697d1f18ca.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:14:41,497 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 20545c8ee2d66091a91ff9697d1f18ca, disabling compactions & flushes 2024-12-02T21:14:41,497 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ppp,1733174081291.20545c8ee2d66091a91ff9697d1f18ca. 2024-12-02T21:14:41,497 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1733174081291.20545c8ee2d66091a91ff9697d1f18ca. 2024-12-02T21:14:41,497 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1733174081291.20545c8ee2d66091a91ff9697d1f18ca. after waiting 0 ms 2024-12-02T21:14:41,497 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1733174081291.20545c8ee2d66091a91ff9697d1f18ca. 2024-12-02T21:14:41,497 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ppp,1733174081291.20545c8ee2d66091a91ff9697d1f18ca. 2024-12-02T21:14:41,497 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 20545c8ee2d66091a91ff9697d1f18ca: Waiting for close lock at 1733174081497Disabling compacts and flushes for region at 1733174081497Disabling writes for close at 1733174081497Writing region close event to WAL at 1733174081497Closed at 1733174081497 2024-12-02T21:14:41,498 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => e42e4828e881f12c74fbedcdb6b00ea2, NAME => 'test,sss,1733174081291.e42e4828e881f12c74fbedcdb6b00ea2.', STARTKEY => 'sss', ENDKEY => 'ttt'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b 2024-12-02T21:14:41,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42827 is added to blk_1073741854_1030 (size=45) 2024-12-02T21:14:41,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741854_1030 (size=45) 2024-12-02T21:14:41,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36203 is added to blk_1073741854_1030 (size=45) 2024-12-02T21:14:41,508 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,qqq,1733174081291.a50bae1d5cb3d03a86dc103b09a0f68b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:14:41,508 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing a50bae1d5cb3d03a86dc103b09a0f68b, disabling compactions & flushes 2024-12-02T21:14:41,508 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,qqq,1733174081291.a50bae1d5cb3d03a86dc103b09a0f68b. 2024-12-02T21:14:41,509 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1733174081291.a50bae1d5cb3d03a86dc103b09a0f68b. 2024-12-02T21:14:41,509 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1733174081291.a50bae1d5cb3d03a86dc103b09a0f68b. after waiting 0 ms 2024-12-02T21:14:41,509 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1733174081291.a50bae1d5cb3d03a86dc103b09a0f68b. 2024-12-02T21:14:41,509 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,qqq,1733174081291.a50bae1d5cb3d03a86dc103b09a0f68b. 2024-12-02T21:14:41,509 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for a50bae1d5cb3d03a86dc103b09a0f68b: Waiting for close lock at 1733174081508Disabling compacts and flushes for region at 1733174081508Disabling writes for close at 1733174081509 (+1 ms)Writing region close event to WAL at 1733174081509Closed at 1733174081509 2024-12-02T21:14:41,510 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d63da23e8236e18b416716ec838dd43e, NAME => 'test,ttt,1733174081291.d63da23e8236e18b416716ec838dd43e.', STARTKEY => 'ttt', ENDKEY => 'uuu'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b 2024-12-02T21:14:41,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36203 is added to blk_1073741856_1032 (size=45) 2024-12-02T21:14:41,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741855_1031 (size=45) 2024-12-02T21:14:41,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42827 is added to blk_1073741856_1032 (size=45) 2024-12-02T21:14:41,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741856_1032 (size=45) 2024-12-02T21:14:41,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36203 is added to blk_1073741855_1031 (size=45) 2024-12-02T21:14:41,517 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,rrr,1733174081291.215753e705337ae11b1b38ad77b6a5cd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:14:41,517 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 215753e705337ae11b1b38ad77b6a5cd, disabling compactions & flushes 2024-12-02T21:14:41,517 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,rrr,1733174081291.215753e705337ae11b1b38ad77b6a5cd. 2024-12-02T21:14:41,517 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1733174081291.215753e705337ae11b1b38ad77b6a5cd. 2024-12-02T21:14:41,518 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1733174081291.215753e705337ae11b1b38ad77b6a5cd. after waiting 0 ms 2024-12-02T21:14:41,518 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1733174081291.215753e705337ae11b1b38ad77b6a5cd. 2024-12-02T21:14:41,518 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,rrr,1733174081291.215753e705337ae11b1b38ad77b6a5cd. 2024-12-02T21:14:41,518 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 215753e705337ae11b1b38ad77b6a5cd: Waiting for close lock at 1733174081517Disabling compacts and flushes for region at 1733174081517Disabling writes for close at 1733174081518 (+1 ms)Writing region close event to WAL at 1733174081518Closed at 1733174081518 2024-12-02T21:14:41,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42827 is added to blk_1073741855_1031 (size=45) 2024-12-02T21:14:41,519 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,sss,1733174081291.e42e4828e881f12c74fbedcdb6b00ea2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:14:41,519 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => e40e25a9323e1d3dad4ec92c578e2f82, NAME => 'test,uuu,1733174081291.e40e25a9323e1d3dad4ec92c578e2f82.', STARTKEY => 'uuu', ENDKEY => 'vvv'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b 2024-12-02T21:14:41,519 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing e42e4828e881f12c74fbedcdb6b00ea2, disabling compactions & flushes 2024-12-02T21:14:41,519 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,sss,1733174081291.e42e4828e881f12c74fbedcdb6b00ea2. 2024-12-02T21:14:41,519 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1733174081291.e42e4828e881f12c74fbedcdb6b00ea2. 2024-12-02T21:14:41,519 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,sss,1733174081291.e42e4828e881f12c74fbedcdb6b00ea2. after waiting 0 ms 2024-12-02T21:14:41,519 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,sss,1733174081291.e42e4828e881f12c74fbedcdb6b00ea2. 2024-12-02T21:14:41,519 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,sss,1733174081291.e42e4828e881f12c74fbedcdb6b00ea2. 2024-12-02T21:14:41,519 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for e42e4828e881f12c74fbedcdb6b00ea2: Waiting for close lock at 1733174081519Disabling compacts and flushes for region at 1733174081519Disabling writes for close at 1733174081519Writing region close event to WAL at 1733174081519Closed at 1733174081519 2024-12-02T21:14:41,520 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 6a2fb6e9d39bf04774472d3b4c70810a, NAME => 'test,vvv,1733174081291.6a2fb6e9d39bf04774472d3b4c70810a.', STARTKEY => 'vvv', ENDKEY => 'www'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b 2024-12-02T21:14:41,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42827 is added to blk_1073741857_1033 (size=45) 2024-12-02T21:14:41,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741857_1033 (size=45) 2024-12-02T21:14:41,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36203 is added to blk_1073741857_1033 (size=45) 2024-12-02T21:14:41,526 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ttt,1733174081291.d63da23e8236e18b416716ec838dd43e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:14:41,527 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing d63da23e8236e18b416716ec838dd43e, disabling compactions & flushes 2024-12-02T21:14:41,527 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ttt,1733174081291.d63da23e8236e18b416716ec838dd43e. 2024-12-02T21:14:41,527 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1733174081291.d63da23e8236e18b416716ec838dd43e. 2024-12-02T21:14:41,527 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1733174081291.d63da23e8236e18b416716ec838dd43e. after waiting 0 ms 2024-12-02T21:14:41,527 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1733174081291.d63da23e8236e18b416716ec838dd43e. 2024-12-02T21:14:41,527 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ttt,1733174081291.d63da23e8236e18b416716ec838dd43e. 2024-12-02T21:14:41,527 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for d63da23e8236e18b416716ec838dd43e: Waiting for close lock at 1733174081527Disabling compacts and flushes for region at 1733174081527Disabling writes for close at 1733174081527Writing region close event to WAL at 1733174081527Closed at 1733174081527 2024-12-02T21:14:41,528 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1a4ea0dec946dcfe96ad162d5bcf5949, NAME => 'test,www,1733174081291.1a4ea0dec946dcfe96ad162d5bcf5949.', STARTKEY => 'www', ENDKEY => 'xxx'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b 2024-12-02T21:14:41,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42827 is added to blk_1073741858_1034 (size=45) 2024-12-02T21:14:41,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36203 is added to blk_1073741858_1034 (size=45) 2024-12-02T21:14:41,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741858_1034 (size=45) 2024-12-02T21:14:41,533 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,uuu,1733174081291.e40e25a9323e1d3dad4ec92c578e2f82.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:14:41,533 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing e40e25a9323e1d3dad4ec92c578e2f82, disabling compactions & flushes 2024-12-02T21:14:41,533 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,uuu,1733174081291.e40e25a9323e1d3dad4ec92c578e2f82. 2024-12-02T21:14:41,533 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1733174081291.e40e25a9323e1d3dad4ec92c578e2f82. 2024-12-02T21:14:41,533 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1733174081291.e40e25a9323e1d3dad4ec92c578e2f82. after waiting 0 ms 2024-12-02T21:14:41,533 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1733174081291.e40e25a9323e1d3dad4ec92c578e2f82. 2024-12-02T21:14:41,533 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,uuu,1733174081291.e40e25a9323e1d3dad4ec92c578e2f82. 2024-12-02T21:14:41,533 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for e40e25a9323e1d3dad4ec92c578e2f82: Waiting for close lock at 1733174081533Disabling compacts and flushes for region at 1733174081533Disabling writes for close at 1733174081533Writing region close event to WAL at 1733174081533Closed at 1733174081533 2024-12-02T21:14:41,534 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 06d131e39cb4f5b02db4ec472d438a26, NAME => 'test,xxx,1733174081291.06d131e39cb4f5b02db4ec472d438a26.', STARTKEY => 'xxx', ENDKEY => 'yyy'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b 2024-12-02T21:14:41,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36203 is added to blk_1073741859_1035 (size=45) 2024-12-02T21:14:41,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741859_1035 (size=45) 2024-12-02T21:14:41,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42827 is added to blk_1073741859_1035 (size=45) 2024-12-02T21:14:41,537 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,vvv,1733174081291.6a2fb6e9d39bf04774472d3b4c70810a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:14:41,537 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 6a2fb6e9d39bf04774472d3b4c70810a, disabling compactions & flushes 2024-12-02T21:14:41,538 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,vvv,1733174081291.6a2fb6e9d39bf04774472d3b4c70810a. 2024-12-02T21:14:41,538 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1733174081291.6a2fb6e9d39bf04774472d3b4c70810a. 2024-12-02T21:14:41,538 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1733174081291.6a2fb6e9d39bf04774472d3b4c70810a. after waiting 0 ms 2024-12-02T21:14:41,538 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1733174081291.6a2fb6e9d39bf04774472d3b4c70810a. 2024-12-02T21:14:41,538 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,vvv,1733174081291.6a2fb6e9d39bf04774472d3b4c70810a. 2024-12-02T21:14:41,538 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 6a2fb6e9d39bf04774472d3b4c70810a: Waiting for close lock at 1733174081537Disabling compacts and flushes for region at 1733174081537Disabling writes for close at 1733174081538 (+1 ms)Writing region close event to WAL at 1733174081538Closed at 1733174081538 2024-12-02T21:14:41,538 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 0875ddf9cbd035a3345be0684b07b2e6, NAME => 'test,yyy,1733174081291.0875ddf9cbd035a3345be0684b07b2e6.', STARTKEY => 'yyy', ENDKEY => 'zzz'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b 2024-12-02T21:14:41,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36203 is added to blk_1073741860_1036 (size=45) 2024-12-02T21:14:41,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42827 is added to blk_1073741860_1036 (size=45) 2024-12-02T21:14:41,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741860_1036 (size=45) 2024-12-02T21:14:41,544 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,www,1733174081291.1a4ea0dec946dcfe96ad162d5bcf5949.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:14:41,545 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 1a4ea0dec946dcfe96ad162d5bcf5949, disabling compactions & flushes 2024-12-02T21:14:41,545 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,www,1733174081291.1a4ea0dec946dcfe96ad162d5bcf5949. 2024-12-02T21:14:41,545 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1733174081291.1a4ea0dec946dcfe96ad162d5bcf5949. 2024-12-02T21:14:41,545 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,www,1733174081291.1a4ea0dec946dcfe96ad162d5bcf5949. after waiting 0 ms 2024-12-02T21:14:41,545 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,www,1733174081291.1a4ea0dec946dcfe96ad162d5bcf5949. 2024-12-02T21:14:41,545 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,www,1733174081291.1a4ea0dec946dcfe96ad162d5bcf5949. 2024-12-02T21:14:41,545 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1a4ea0dec946dcfe96ad162d5bcf5949: Waiting for close lock at 1733174081545Disabling compacts and flushes for region at 1733174081545Disabling writes for close at 1733174081545Writing region close event to WAL at 1733174081545Closed at 1733174081545 2024-12-02T21:14:41,546 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a36d8f99c2aef77f88827585926ad989, NAME => 'test,zzz,1733174081291.a36d8f99c2aef77f88827585926ad989.', STARTKEY => 'zzz', ENDKEY => ''}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b 2024-12-02T21:14:41,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42827 is added to blk_1073741862_1038 (size=45) 2024-12-02T21:14:41,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36203 is added to blk_1073741862_1038 (size=45) 2024-12-02T21:14:41,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741862_1038 (size=45) 2024-12-02T21:14:41,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42827 is added to blk_1073741861_1037 (size=45) 2024-12-02T21:14:41,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36203 is added to blk_1073741861_1037 (size=45) 2024-12-02T21:14:41,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741861_1037 (size=45) 2024-12-02T21:14:41,555 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,yyy,1733174081291.0875ddf9cbd035a3345be0684b07b2e6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:14:41,555 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 0875ddf9cbd035a3345be0684b07b2e6, disabling compactions & flushes 2024-12-02T21:14:41,555 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,yyy,1733174081291.0875ddf9cbd035a3345be0684b07b2e6. 2024-12-02T21:14:41,555 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1733174081291.0875ddf9cbd035a3345be0684b07b2e6. 2024-12-02T21:14:41,555 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1733174081291.0875ddf9cbd035a3345be0684b07b2e6. after waiting 0 ms 2024-12-02T21:14:41,555 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1733174081291.0875ddf9cbd035a3345be0684b07b2e6. 2024-12-02T21:14:41,555 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,yyy,1733174081291.0875ddf9cbd035a3345be0684b07b2e6. 2024-12-02T21:14:41,555 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 0875ddf9cbd035a3345be0684b07b2e6: Waiting for close lock at 1733174081555Disabling compacts and flushes for region at 1733174081555Disabling writes for close at 1733174081555Writing region close event to WAL at 1733174081555Closed at 1733174081555 2024-12-02T21:14:41,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42827 is added to blk_1073741863_1039 (size=42) 2024-12-02T21:14:41,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36203 is added to blk_1073741863_1039 (size=42) 2024-12-02T21:14:41,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741863_1039 (size=42) 2024-12-02T21:14:41,560 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,zzz,1733174081291.a36d8f99c2aef77f88827585926ad989.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:14:41,560 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing a36d8f99c2aef77f88827585926ad989, disabling compactions & flushes 2024-12-02T21:14:41,560 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,zzz,1733174081291.a36d8f99c2aef77f88827585926ad989. 2024-12-02T21:14:41,560 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1733174081291.a36d8f99c2aef77f88827585926ad989. 2024-12-02T21:14:41,560 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1733174081291.a36d8f99c2aef77f88827585926ad989. after waiting 0 ms 2024-12-02T21:14:41,560 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1733174081291.a36d8f99c2aef77f88827585926ad989. 2024-12-02T21:14:41,560 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,zzz,1733174081291.a36d8f99c2aef77f88827585926ad989. 2024-12-02T21:14:41,561 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for a36d8f99c2aef77f88827585926ad989: Waiting for close lock at 1733174081560Disabling compacts and flushes for region at 1733174081560Disabling writes for close at 1733174081560Writing region close event to WAL at 1733174081560Closed at 1733174081560 2024-12-02T21:14:41,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T21:14:41,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T21:14:41,954 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,xxx,1733174081291.06d131e39cb4f5b02db4ec472d438a26.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:14:41,955 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 06d131e39cb4f5b02db4ec472d438a26, disabling compactions & flushes 2024-12-02T21:14:41,955 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,xxx,1733174081291.06d131e39cb4f5b02db4ec472d438a26. 2024-12-02T21:14:41,955 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1733174081291.06d131e39cb4f5b02db4ec472d438a26. 2024-12-02T21:14:41,955 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1733174081291.06d131e39cb4f5b02db4ec472d438a26. after waiting 0 ms 2024-12-02T21:14:41,955 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1733174081291.06d131e39cb4f5b02db4ec472d438a26. 2024-12-02T21:14:41,955 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,xxx,1733174081291.06d131e39cb4f5b02db4ec472d438a26. 2024-12-02T21:14:41,955 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 06d131e39cb4f5b02db4ec472d438a26: Waiting for close lock at 1733174081955Disabling compacts and flushes for region at 1733174081955Disabling writes for close at 1733174081955Writing region close event to WAL at 1733174081955Closed at 1733174081955 2024-12-02T21:14:41,959 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ADD_TO_META 2024-12-02T21:14:41,964 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,bbb,1733174081291.5ed86f04f519cc5dccd1b6d41be3da7d.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733174081959"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733174081959"}]},"ts":"1733174081959"} 2024-12-02T21:14:41,965 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ccc,1733174081291.0517912ff18e81317bb5474cefaca025.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733174081959"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733174081959"}]},"ts":"1733174081959"} 2024-12-02T21:14:41,965 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,,1733174081291.85ca3a9a09d75c0941c58f5b83584a8e.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1733174081959"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733174081959"}]},"ts":"1733174081959"} 2024-12-02T21:14:41,965 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ddd,1733174081291.ff611163b4ab91c50d940651a0065c34.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733174081959"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733174081959"}]},"ts":"1733174081959"} 2024-12-02T21:14:41,965 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,eee,1733174081291.120cf3776e92a15d3276f11911c4042f.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733174081959"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733174081959"}]},"ts":"1733174081959"} 2024-12-02T21:14:41,965 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,fff,1733174081291.e87d99b937228303e28b7ea0373ddc70.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733174081959"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733174081959"}]},"ts":"1733174081959"} 2024-12-02T21:14:41,965 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,hhh,1733174081291.9950441eda7548fa66872811a13f5bf0.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733174081959"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733174081959"}]},"ts":"1733174081959"} 2024-12-02T21:14:41,966 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ggg,1733174081291.0dfd4169e4c254c51bcc70a472f99ecf.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733174081959"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733174081959"}]},"ts":"1733174081959"} 2024-12-02T21:14:41,966 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,iii,1733174081291.8e039115fcc0d0a54e2d03efed7e05de.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733174081959"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733174081959"}]},"ts":"1733174081959"} 2024-12-02T21:14:41,966 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,kkk,1733174081291.a524d88be9ac29b416f0a6c04627fa1c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733174081959"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733174081959"}]},"ts":"1733174081959"} 2024-12-02T21:14:41,966 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,lll,1733174081291.0d0be65f6ffd7ebb4c8b2fee37042136.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733174081959"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733174081959"}]},"ts":"1733174081959"} 2024-12-02T21:14:41,966 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,jjj,1733174081291.5d60c566b39516dd875aaf2bd7def402.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733174081959"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733174081959"}]},"ts":"1733174081959"} 2024-12-02T21:14:41,966 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,mmm,1733174081291.292fd0b4e0f07516cc25c069181e3111.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733174081959"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733174081959"}]},"ts":"1733174081959"} 2024-12-02T21:14:41,966 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,nnn,1733174081291.87ba365a3e5d028c9193155af96aeb60.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733174081959"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733174081959"}]},"ts":"1733174081959"} 2024-12-02T21:14:41,967 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ooo,1733174081291.4197cd4f7ea2f3d6256be6836e9834da.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733174081959"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733174081959"}]},"ts":"1733174081959"} 2024-12-02T21:14:41,967 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ppp,1733174081291.20545c8ee2d66091a91ff9697d1f18ca.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733174081959"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733174081959"}]},"ts":"1733174081959"} 2024-12-02T21:14:41,967 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,qqq,1733174081291.a50bae1d5cb3d03a86dc103b09a0f68b.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733174081959"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733174081959"}]},"ts":"1733174081959"} 2024-12-02T21:14:41,967 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,rrr,1733174081291.215753e705337ae11b1b38ad77b6a5cd.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733174081959"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733174081959"}]},"ts":"1733174081959"} 2024-12-02T21:14:41,967 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,sss,1733174081291.e42e4828e881f12c74fbedcdb6b00ea2.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733174081959"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733174081959"}]},"ts":"1733174081959"} 2024-12-02T21:14:41,967 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ttt,1733174081291.d63da23e8236e18b416716ec838dd43e.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733174081959"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733174081959"}]},"ts":"1733174081959"} 2024-12-02T21:14:41,967 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,uuu,1733174081291.e40e25a9323e1d3dad4ec92c578e2f82.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733174081959"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733174081959"}]},"ts":"1733174081959"} 2024-12-02T21:14:41,968 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,vvv,1733174081291.6a2fb6e9d39bf04774472d3b4c70810a.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733174081959"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733174081959"}]},"ts":"1733174081959"} 2024-12-02T21:14:41,968 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,www,1733174081291.1a4ea0dec946dcfe96ad162d5bcf5949.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733174081959"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733174081959"}]},"ts":"1733174081959"} 2024-12-02T21:14:41,968 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,yyy,1733174081291.0875ddf9cbd035a3345be0684b07b2e6.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733174081959"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733174081959"}]},"ts":"1733174081959"} 2024-12-02T21:14:41,968 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,zzz,1733174081291.a36d8f99c2aef77f88827585926ad989.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1733174081959"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733174081959"}]},"ts":"1733174081959"} 2024-12-02T21:14:41,968 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,xxx,1733174081291.06d131e39cb4f5b02db4ec472d438a26.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733174081959"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733174081959"}]},"ts":"1733174081959"} 2024-12-02T21:14:42,005 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 26 regions to meta. 2024-12-02T21:14:42,007 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-02T21:14:42,010 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733174082007"}]},"ts":"1733174082007"} 2024-12-02T21:14:42,015 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLING in hbase:meta 2024-12-02T21:14:42,016 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {87c3fdb6c570=0} racks are {/default-rack=0} 2024-12-02T21:14:42,030 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-02T21:14:42,031 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-02T21:14:42,031 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-02T21:14:42,031 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-02T21:14:42,031 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-02T21:14:42,031 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-02T21:14:42,031 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-02T21:14:42,031 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-02T21:14:42,031 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-02T21:14:42,031 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-02T21:14:42,033 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=85ca3a9a09d75c0941c58f5b83584a8e, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5ed86f04f519cc5dccd1b6d41be3da7d, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ff611163b4ab91c50d940651a0065c34, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=120cf3776e92a15d3276f11911c4042f, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e87d99b937228303e28b7ea0373ddc70, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0dfd4169e4c254c51bcc70a472f99ecf, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9950441eda7548fa66872811a13f5bf0, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8e039115fcc0d0a54e2d03efed7e05de, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5d60c566b39516dd875aaf2bd7def402, ASSIGN}, {pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a524d88be9ac29b416f0a6c04627fa1c, ASSIGN}, {pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0d0be65f6ffd7ebb4c8b2fee37042136, ASSIGN}, {pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=292fd0b4e0f07516cc25c069181e3111, ASSIGN}, {pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=87ba365a3e5d028c9193155af96aeb60, ASSIGN}, {pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4197cd4f7ea2f3d6256be6836e9834da, ASSIGN}, {pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=20545c8ee2d66091a91ff9697d1f18ca, ASSIGN}, {pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b, ASSIGN}, {pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=215753e705337ae11b1b38ad77b6a5cd, ASSIGN}, {pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e42e4828e881f12c74fbedcdb6b00ea2, ASSIGN}, {pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d63da23e8236e18b416716ec838dd43e, ASSIGN}, {pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e40e25a9323e1d3dad4ec92c578e2f82, ASSIGN}, {pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6a2fb6e9d39bf04774472d3b4c70810a, ASSIGN}, {pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN}, {pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN}, {pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0875ddf9cbd035a3345be0684b07b2e6, ASSIGN}, {pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a36d8f99c2aef77f88827585926ad989, ASSIGN}] 2024-12-02T21:14:42,040 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a36d8f99c2aef77f88827585926ad989, ASSIGN 2024-12-02T21:14:42,041 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a36d8f99c2aef77f88827585926ad989, ASSIGN; state=OFFLINE, location=87c3fdb6c570,40413,1733174078061; forceNewPlan=false, retain=false 2024-12-02T21:14:42,043 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0875ddf9cbd035a3345be0684b07b2e6, ASSIGN 2024-12-02T21:14:42,045 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=0875ddf9cbd035a3345be0684b07b2e6, ASSIGN; state=OFFLINE, location=87c3fdb6c570,36585,1733174078182; forceNewPlan=false, retain=false 2024-12-02T21:14:42,046 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN 2024-12-02T21:14:42,047 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN; state=OFFLINE, location=87c3fdb6c570,33459,1733174078298; forceNewPlan=false, retain=false 2024-12-02T21:14:42,049 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN 2024-12-02T21:14:42,050 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN; state=OFFLINE, location=87c3fdb6c570,33459,1733174078298; forceNewPlan=false, retain=false 2024-12-02T21:14:42,051 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6a2fb6e9d39bf04774472d3b4c70810a, ASSIGN 2024-12-02T21:14:42,053 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=6a2fb6e9d39bf04774472d3b4c70810a, ASSIGN; state=OFFLINE, location=87c3fdb6c570,40413,1733174078061; forceNewPlan=false, retain=false 2024-12-02T21:14:42,054 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e40e25a9323e1d3dad4ec92c578e2f82, ASSIGN 2024-12-02T21:14:42,056 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=e40e25a9323e1d3dad4ec92c578e2f82, ASSIGN; state=OFFLINE, location=87c3fdb6c570,36585,1733174078182; forceNewPlan=false, retain=false 2024-12-02T21:14:42,057 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d63da23e8236e18b416716ec838dd43e, ASSIGN 2024-12-02T21:14:42,059 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d63da23e8236e18b416716ec838dd43e, ASSIGN; state=OFFLINE, location=87c3fdb6c570,33459,1733174078298; forceNewPlan=false, retain=false 2024-12-02T21:14:42,060 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e42e4828e881f12c74fbedcdb6b00ea2, ASSIGN 2024-12-02T21:14:42,061 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=e42e4828e881f12c74fbedcdb6b00ea2, ASSIGN; state=OFFLINE, location=87c3fdb6c570,40413,1733174078061; forceNewPlan=false, retain=false 2024-12-02T21:14:42,062 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=215753e705337ae11b1b38ad77b6a5cd, ASSIGN 2024-12-02T21:14:42,063 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=215753e705337ae11b1b38ad77b6a5cd, ASSIGN; state=OFFLINE, location=87c3fdb6c570,36585,1733174078182; forceNewPlan=false, retain=false 2024-12-02T21:14:42,065 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b, ASSIGN 2024-12-02T21:14:42,067 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b, ASSIGN; state=OFFLINE, location=87c3fdb6c570,33459,1733174078298; forceNewPlan=false, retain=false 2024-12-02T21:14:42,068 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=20545c8ee2d66091a91ff9697d1f18ca, ASSIGN 2024-12-02T21:14:42,070 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=20545c8ee2d66091a91ff9697d1f18ca, ASSIGN; state=OFFLINE, location=87c3fdb6c570,40413,1733174078061; forceNewPlan=false, retain=false 2024-12-02T21:14:42,071 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4197cd4f7ea2f3d6256be6836e9834da, ASSIGN 2024-12-02T21:14:42,073 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4197cd4f7ea2f3d6256be6836e9834da, ASSIGN; state=OFFLINE, location=87c3fdb6c570,36585,1733174078182; forceNewPlan=false, retain=false 2024-12-02T21:14:42,074 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=87ba365a3e5d028c9193155af96aeb60, ASSIGN 2024-12-02T21:14:42,076 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=87ba365a3e5d028c9193155af96aeb60, ASSIGN; state=OFFLINE, location=87c3fdb6c570,33459,1733174078298; forceNewPlan=false, retain=false 2024-12-02T21:14:42,077 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=292fd0b4e0f07516cc25c069181e3111, ASSIGN 2024-12-02T21:14:42,078 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=292fd0b4e0f07516cc25c069181e3111, ASSIGN; state=OFFLINE, location=87c3fdb6c570,40413,1733174078061; forceNewPlan=false, retain=false 2024-12-02T21:14:42,080 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0d0be65f6ffd7ebb4c8b2fee37042136, ASSIGN 2024-12-02T21:14:42,081 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=0d0be65f6ffd7ebb4c8b2fee37042136, ASSIGN; state=OFFLINE, location=87c3fdb6c570,33459,1733174078298; forceNewPlan=false, retain=false 2024-12-02T21:14:42,082 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a524d88be9ac29b416f0a6c04627fa1c, ASSIGN 2024-12-02T21:14:42,083 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a524d88be9ac29b416f0a6c04627fa1c, ASSIGN; state=OFFLINE, location=87c3fdb6c570,40413,1733174078061; forceNewPlan=false, retain=false 2024-12-02T21:14:42,085 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5d60c566b39516dd875aaf2bd7def402, ASSIGN 2024-12-02T21:14:42,091 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=5d60c566b39516dd875aaf2bd7def402, ASSIGN; state=OFFLINE, location=87c3fdb6c570,36585,1733174078182; forceNewPlan=false, retain=false 2024-12-02T21:14:42,093 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8e039115fcc0d0a54e2d03efed7e05de, ASSIGN 2024-12-02T21:14:42,094 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8e039115fcc0d0a54e2d03efed7e05de, ASSIGN; state=OFFLINE, location=87c3fdb6c570,36585,1733174078182; forceNewPlan=false, retain=false 2024-12-02T21:14:42,096 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9950441eda7548fa66872811a13f5bf0, ASSIGN 2024-12-02T21:14:42,098 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=9950441eda7548fa66872811a13f5bf0, ASSIGN; state=OFFLINE, location=87c3fdb6c570,40413,1733174078061; forceNewPlan=false, retain=false 2024-12-02T21:14:42,100 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0dfd4169e4c254c51bcc70a472f99ecf, ASSIGN 2024-12-02T21:14:42,102 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=0dfd4169e4c254c51bcc70a472f99ecf, ASSIGN; state=OFFLINE, location=87c3fdb6c570,33459,1733174078298; forceNewPlan=false, retain=false 2024-12-02T21:14:42,103 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e87d99b937228303e28b7ea0373ddc70, ASSIGN 2024-12-02T21:14:42,105 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=e87d99b937228303e28b7ea0373ddc70, ASSIGN; state=OFFLINE, location=87c3fdb6c570,36585,1733174078182; forceNewPlan=false, retain=false 2024-12-02T21:14:42,107 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=120cf3776e92a15d3276f11911c4042f, ASSIGN 2024-12-02T21:14:42,108 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=120cf3776e92a15d3276f11911c4042f, ASSIGN; state=OFFLINE, location=87c3fdb6c570,33459,1733174078298; forceNewPlan=false, retain=false 2024-12-02T21:14:42,110 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ff611163b4ab91c50d940651a0065c34, ASSIGN 2024-12-02T21:14:42,111 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ff611163b4ab91c50d940651a0065c34, ASSIGN; state=OFFLINE, location=87c3fdb6c570,40413,1733174078061; forceNewPlan=false, retain=false 2024-12-02T21:14:42,112 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN 2024-12-02T21:14:42,113 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN; state=OFFLINE, location=87c3fdb6c570,33459,1733174078298; forceNewPlan=false, retain=false 2024-12-02T21:14:42,115 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5ed86f04f519cc5dccd1b6d41be3da7d, ASSIGN 2024-12-02T21:14:42,116 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=5ed86f04f519cc5dccd1b6d41be3da7d, ASSIGN; state=OFFLINE, location=87c3fdb6c570,40413,1733174078061; forceNewPlan=false, retain=false 2024-12-02T21:14:42,118 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=85ca3a9a09d75c0941c58f5b83584a8e, ASSIGN 2024-12-02T21:14:42,119 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=85ca3a9a09d75c0941c58f5b83584a8e, ASSIGN; state=OFFLINE, location=87c3fdb6c570,36585,1733174078182; forceNewPlan=false, retain=false 2024-12-02T21:14:42,194 INFO [87c3fdb6c570:42045 {}] balancer.BaseLoadBalancer(388): Reassigned 26 regions. 26 retained the pre-restart assignment. 2024-12-02T21:14:42,195 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=a36d8f99c2aef77f88827585926ad989, regionState=OPENING, regionLocation=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:42,196 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=6a2fb6e9d39bf04774472d3b4c70810a, regionState=OPENING, regionLocation=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:42,197 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=e42e4828e881f12c74fbedcdb6b00ea2, regionState=OPENING, regionLocation=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:42,198 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=20545c8ee2d66091a91ff9697d1f18ca, regionState=OPENING, regionLocation=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:42,198 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=292fd0b4e0f07516cc25c069181e3111, regionState=OPENING, regionLocation=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:42,199 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=a524d88be9ac29b416f0a6c04627fa1c, regionState=OPENING, regionLocation=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:42,199 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=9950441eda7548fa66872811a13f5bf0, regionState=OPENING, regionLocation=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:42,200 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=ff611163b4ab91c50d940651a0065c34, regionState=OPENING, regionLocation=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:42,201 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=5ed86f04f519cc5dccd1b6d41be3da7d, regionState=OPENING, regionLocation=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:42,202 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=0875ddf9cbd035a3345be0684b07b2e6, regionState=OPENING, regionLocation=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:42,202 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a36d8f99c2aef77f88827585926ad989, ASSIGN because future has completed 2024-12-02T21:14:42,203 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=e40e25a9323e1d3dad4ec92c578e2f82, regionState=OPENING, regionLocation=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:42,203 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure a36d8f99c2aef77f88827585926ad989, server=87c3fdb6c570,40413,1733174078061}] 2024-12-02T21:14:42,204 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=6a2fb6e9d39bf04774472d3b4c70810a, ASSIGN because future has completed 2024-12-02T21:14:42,206 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=e42e4828e881f12c74fbedcdb6b00ea2, ASSIGN because future has completed 2024-12-02T21:14:42,208 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=20545c8ee2d66091a91ff9697d1f18ca, ASSIGN because future has completed 2024-12-02T21:14:42,210 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=292fd0b4e0f07516cc25c069181e3111, ASSIGN because future has completed 2024-12-02T21:14:42,211 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a524d88be9ac29b416f0a6c04627fa1c, ASSIGN because future has completed 2024-12-02T21:14:42,213 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure a524d88be9ac29b416f0a6c04627fa1c, server=87c3fdb6c570,40413,1733174078061}] 2024-12-02T21:14:42,213 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=9950441eda7548fa66872811a13f5bf0, ASSIGN because future has completed 2024-12-02T21:14:42,215 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ff611163b4ab91c50d940651a0065c34, ASSIGN because future has completed 2024-12-02T21:14:42,217 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=5ed86f04f519cc5dccd1b6d41be3da7d, ASSIGN because future has completed 2024-12-02T21:14:42,219 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=0875ddf9cbd035a3345be0684b07b2e6, ASSIGN because future has completed 2024-12-02T21:14:42,219 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5ed86f04f519cc5dccd1b6d41be3da7d, server=87c3fdb6c570,40413,1733174078061}] 2024-12-02T21:14:42,222 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=e40e25a9323e1d3dad4ec92c578e2f82, ASSIGN because future has completed 2024-12-02T21:14:42,225 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure e40e25a9323e1d3dad4ec92c578e2f82, server=87c3fdb6c570,36585,1733174078182}] 2024-12-02T21:14:42,230 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0875ddf9cbd035a3345be0684b07b2e6, server=87c3fdb6c570,36585,1733174078182}] 2024-12-02T21:14:42,235 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure ff611163b4ab91c50d940651a0065c34, server=87c3fdb6c570,40413,1733174078061}] 2024-12-02T21:14:42,240 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9950441eda7548fa66872811a13f5bf0, server=87c3fdb6c570,40413,1733174078061}] 2024-12-02T21:14:42,245 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 292fd0b4e0f07516cc25c069181e3111, server=87c3fdb6c570,40413,1733174078061}] 2024-12-02T21:14:42,249 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 20545c8ee2d66091a91ff9697d1f18ca, server=87c3fdb6c570,40413,1733174078061}] 2024-12-02T21:14:42,254 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure e42e4828e881f12c74fbedcdb6b00ea2, server=87c3fdb6c570,40413,1733174078061}] 2024-12-02T21:14:42,258 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6a2fb6e9d39bf04774472d3b4c70810a, server=87c3fdb6c570,40413,1733174078061}] 2024-12-02T21:14:42,262 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=215753e705337ae11b1b38ad77b6a5cd, regionState=OPENING, regionLocation=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:42,263 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=4197cd4f7ea2f3d6256be6836e9834da, regionState=OPENING, regionLocation=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:42,263 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=5d60c566b39516dd875aaf2bd7def402, regionState=OPENING, regionLocation=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:42,264 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=8e039115fcc0d0a54e2d03efed7e05de, regionState=OPENING, regionLocation=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:42,264 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=e87d99b937228303e28b7ea0373ddc70, regionState=OPENING, regionLocation=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:42,265 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=85ca3a9a09d75c0941c58f5b83584a8e, regionState=OPENING, regionLocation=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:42,265 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=215753e705337ae11b1b38ad77b6a5cd, ASSIGN because future has completed 2024-12-02T21:14:42,265 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=06d131e39cb4f5b02db4ec472d438a26, regionState=OPENING, regionLocation=87c3fdb6c570,33459,1733174078298 2024-12-02T21:14:42,266 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 215753e705337ae11b1b38ad77b6a5cd, server=87c3fdb6c570,36585,1733174078182}] 2024-12-02T21:14:42,267 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4197cd4f7ea2f3d6256be6836e9834da, ASSIGN because future has completed 2024-12-02T21:14:42,269 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=5d60c566b39516dd875aaf2bd7def402, ASSIGN because future has completed 2024-12-02T21:14:42,271 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8e039115fcc0d0a54e2d03efed7e05de, ASSIGN because future has completed 2024-12-02T21:14:42,271 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5d60c566b39516dd875aaf2bd7def402, server=87c3fdb6c570,36585,1733174078182}] 2024-12-02T21:14:42,272 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=e87d99b937228303e28b7ea0373ddc70, ASSIGN because future has completed 2024-12-02T21:14:42,275 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure e87d99b937228303e28b7ea0373ddc70, server=87c3fdb6c570,36585,1733174078182}] 2024-12-02T21:14:42,278 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=85ca3a9a09d75c0941c58f5b83584a8e, ASSIGN because future has completed 2024-12-02T21:14:42,281 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN because future has completed 2024-12-02T21:14:42,283 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06d131e39cb4f5b02db4ec472d438a26, server=87c3fdb6c570,33459,1733174078298}] 2024-12-02T21:14:42,292 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 85ca3a9a09d75c0941c58f5b83584a8e, server=87c3fdb6c570,36585,1733174078182}] 2024-12-02T21:14:42,299 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8e039115fcc0d0a54e2d03efed7e05de, server=87c3fdb6c570,36585,1733174078182}] 2024-12-02T21:14:42,304 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4197cd4f7ea2f3d6256be6836e9834da, server=87c3fdb6c570,36585,1733174078182}] 2024-12-02T21:14:42,307 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=1a4ea0dec946dcfe96ad162d5bcf5949, regionState=OPENING, regionLocation=87c3fdb6c570,33459,1733174078298 2024-12-02T21:14:42,308 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=d63da23e8236e18b416716ec838dd43e, regionState=OPENING, regionLocation=87c3fdb6c570,33459,1733174078298 2024-12-02T21:14:42,309 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=a50bae1d5cb3d03a86dc103b09a0f68b, regionState=OPENING, regionLocation=87c3fdb6c570,33459,1733174078298 2024-12-02T21:14:42,309 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=87ba365a3e5d028c9193155af96aeb60, regionState=OPENING, regionLocation=87c3fdb6c570,33459,1733174078298 2024-12-02T21:14:42,310 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=0d0be65f6ffd7ebb4c8b2fee37042136, regionState=OPENING, regionLocation=87c3fdb6c570,33459,1733174078298 2024-12-02T21:14:42,311 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN because future has completed 2024-12-02T21:14:42,311 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=0dfd4169e4c254c51bcc70a472f99ecf, regionState=OPENING, regionLocation=87c3fdb6c570,33459,1733174078298 2024-12-02T21:14:42,311 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1a4ea0dec946dcfe96ad162d5bcf5949, server=87c3fdb6c570,33459,1733174078298}] 2024-12-02T21:14:42,313 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d63da23e8236e18b416716ec838dd43e, ASSIGN because future has completed 2024-12-02T21:14:42,314 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b, ASSIGN because future has completed 2024-12-02T21:14:42,315 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=87ba365a3e5d028c9193155af96aeb60, ASSIGN because future has completed 2024-12-02T21:14:42,317 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=0d0be65f6ffd7ebb4c8b2fee37042136, ASSIGN because future has completed 2024-12-02T21:14:42,317 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0d0be65f6ffd7ebb4c8b2fee37042136, server=87c3fdb6c570,33459,1733174078298}] 2024-12-02T21:14:42,318 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=0dfd4169e4c254c51bcc70a472f99ecf, ASSIGN because future has completed 2024-12-02T21:14:42,321 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0dfd4169e4c254c51bcc70a472f99ecf, server=87c3fdb6c570,33459,1733174078298}] 2024-12-02T21:14:42,325 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 87ba365a3e5d028c9193155af96aeb60, server=87c3fdb6c570,33459,1733174078298}] 2024-12-02T21:14:42,329 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure a50bae1d5cb3d03a86dc103b09a0f68b, server=87c3fdb6c570,33459,1733174078298}] 2024-12-02T21:14:42,332 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,33459,1733174078298}] 2024-12-02T21:14:42,335 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=120cf3776e92a15d3276f11911c4042f, regionState=OPENING, regionLocation=87c3fdb6c570,33459,1733174078298 2024-12-02T21:14:42,336 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=0517912ff18e81317bb5474cefaca025, regionState=OPENING, regionLocation=87c3fdb6c570,33459,1733174078298 2024-12-02T21:14:42,338 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=120cf3776e92a15d3276f11911c4042f, ASSIGN because future has completed 2024-12-02T21:14:42,338 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 120cf3776e92a15d3276f11911c4042f, server=87c3fdb6c570,33459,1733174078298}] 2024-12-02T21:14:42,339 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN because future has completed 2024-12-02T21:14:42,342 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0517912ff18e81317bb5474cefaca025, server=87c3fdb6c570,33459,1733174078298}] 2024-12-02T21:14:42,358 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-02T21:14:42,360 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38435, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-02T21:14:42,366 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open test,hhh,1733174081291.9950441eda7548fa66872811a13f5bf0. 2024-12-02T21:14:42,366 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 9950441eda7548fa66872811a13f5bf0, NAME => 'test,hhh,1733174081291.9950441eda7548fa66872811a13f5bf0.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-12-02T21:14:42,367 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 9950441eda7548fa66872811a13f5bf0 2024-12-02T21:14:42,367 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated test,hhh,1733174081291.9950441eda7548fa66872811a13f5bf0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:14:42,367 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 9950441eda7548fa66872811a13f5bf0 2024-12-02T21:14:42,367 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 9950441eda7548fa66872811a13f5bf0 2024-12-02T21:14:42,370 INFO [StoreOpener-9950441eda7548fa66872811a13f5bf0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 9950441eda7548fa66872811a13f5bf0 2024-12-02T21:14:42,372 INFO [StoreOpener-9950441eda7548fa66872811a13f5bf0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9950441eda7548fa66872811a13f5bf0 columnFamilyName family 2024-12-02T21:14:42,372 DEBUG [StoreOpener-9950441eda7548fa66872811a13f5bf0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:14:42,374 INFO [StoreOpener-9950441eda7548fa66872811a13f5bf0-1 {}] regionserver.HStore(327): Store=9950441eda7548fa66872811a13f5bf0/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T21:14:42,374 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 9950441eda7548fa66872811a13f5bf0 2024-12-02T21:14:42,375 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/9950441eda7548fa66872811a13f5bf0 2024-12-02T21:14:42,376 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/9950441eda7548fa66872811a13f5bf0 2024-12-02T21:14:42,376 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 9950441eda7548fa66872811a13f5bf0 2024-12-02T21:14:42,376 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 9950441eda7548fa66872811a13f5bf0 2024-12-02T21:14:42,379 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 9950441eda7548fa66872811a13f5bf0 2024-12-02T21:14:42,379 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-02T21:14:42,381 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35781, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-02T21:14:42,382 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/9950441eda7548fa66872811a13f5bf0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T21:14:42,383 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 9950441eda7548fa66872811a13f5bf0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61386756, jitterRate=-0.08526605367660522}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T21:14:42,383 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9950441eda7548fa66872811a13f5bf0 2024-12-02T21:14:42,384 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 9950441eda7548fa66872811a13f5bf0: Running coprocessor pre-open hook at 1733174082367Writing region info on filesystem at 1733174082367Initializing all the Stores at 1733174082369 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174082369Cleaning up temporary data from old regions at 1733174082376 (+7 ms)Running coprocessor post-open hooks at 1733174082383 (+7 ms)Region opened successfully at 1733174082384 (+1 ms) 2024-12-02T21:14:42,386 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1733174081291.9950441eda7548fa66872811a13f5bf0., pid=37, masterSystemTime=1733174082358 2024-12-02T21:14:42,387 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(132): Open test,rrr,1733174081291.215753e705337ae11b1b38ad77b6a5cd. 2024-12-02T21:14:42,387 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7752): Opening region: {ENCODED => 215753e705337ae11b1b38ad77b6a5cd, NAME => 'test,rrr,1733174081291.215753e705337ae11b1b38ad77b6a5cd.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-12-02T21:14:42,387 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 215753e705337ae11b1b38ad77b6a5cd 2024-12-02T21:14:42,387 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(898): Instantiated test,rrr,1733174081291.215753e705337ae11b1b38ad77b6a5cd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:14:42,388 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7794): checking encryption for 215753e705337ae11b1b38ad77b6a5cd 2024-12-02T21:14:42,388 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7797): checking classloading for 215753e705337ae11b1b38ad77b6a5cd 2024-12-02T21:14:42,390 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1733174081291.9950441eda7548fa66872811a13f5bf0. 2024-12-02T21:14:42,391 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened test,hhh,1733174081291.9950441eda7548fa66872811a13f5bf0. 2024-12-02T21:14:42,391 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open test,sss,1733174081291.e42e4828e881f12c74fbedcdb6b00ea2. 2024-12-02T21:14:42,391 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => e42e4828e881f12c74fbedcdb6b00ea2, NAME => 'test,sss,1733174081291.e42e4828e881f12c74fbedcdb6b00ea2.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-12-02T21:14:42,391 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test e42e4828e881f12c74fbedcdb6b00ea2 2024-12-02T21:14:42,392 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=9950441eda7548fa66872811a13f5bf0, regionState=OPEN, openSeqNum=2, regionLocation=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:42,392 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated test,sss,1733174081291.e42e4828e881f12c74fbedcdb6b00ea2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:14:42,392 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for e42e4828e881f12c74fbedcdb6b00ea2 2024-12-02T21:14:42,392 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for e42e4828e881f12c74fbedcdb6b00ea2 2024-12-02T21:14:42,395 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9950441eda7548fa66872811a13f5bf0, server=87c3fdb6c570,40413,1733174078061 because future has completed 2024-12-02T21:14:42,396 INFO [StoreOpener-e42e4828e881f12c74fbedcdb6b00ea2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region e42e4828e881f12c74fbedcdb6b00ea2 2024-12-02T21:14:42,398 INFO [StoreOpener-e42e4828e881f12c74fbedcdb6b00ea2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e42e4828e881f12c74fbedcdb6b00ea2 columnFamilyName family 2024-12-02T21:14:42,399 DEBUG [StoreOpener-e42e4828e881f12c74fbedcdb6b00ea2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:14:42,399 INFO [StoreOpener-e42e4828e881f12c74fbedcdb6b00ea2-1 {}] regionserver.HStore(327): Store=e42e4828e881f12c74fbedcdb6b00ea2/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T21:14:42,400 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for e42e4828e881f12c74fbedcdb6b00ea2 2024-12-02T21:14:42,400 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=12 2024-12-02T21:14:42,400 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure 9950441eda7548fa66872811a13f5bf0, server=87c3fdb6c570,40413,1733174078061 in 157 msec 2024-12-02T21:14:42,400 INFO [StoreOpener-215753e705337ae11b1b38ad77b6a5cd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 215753e705337ae11b1b38ad77b6a5cd 2024-12-02T21:14:42,401 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/e42e4828e881f12c74fbedcdb6b00ea2 2024-12-02T21:14:42,402 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/e42e4828e881f12c74fbedcdb6b00ea2 2024-12-02T21:14:42,402 INFO [StoreOpener-215753e705337ae11b1b38ad77b6a5cd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 215753e705337ae11b1b38ad77b6a5cd columnFamilyName family 2024-12-02T21:14:42,402 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for e42e4828e881f12c74fbedcdb6b00ea2 2024-12-02T21:14:42,402 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for e42e4828e881f12c74fbedcdb6b00ea2 2024-12-02T21:14:42,402 DEBUG [StoreOpener-215753e705337ae11b1b38ad77b6a5cd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:14:42,403 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=9950441eda7548fa66872811a13f5bf0, ASSIGN in 367 msec 2024-12-02T21:14:42,403 INFO [StoreOpener-215753e705337ae11b1b38ad77b6a5cd-1 {}] regionserver.HStore(327): Store=215753e705337ae11b1b38ad77b6a5cd/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T21:14:42,403 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1038): replaying wal for 215753e705337ae11b1b38ad77b6a5cd 2024-12-02T21:14:42,404 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/215753e705337ae11b1b38ad77b6a5cd 2024-12-02T21:14:42,405 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/215753e705337ae11b1b38ad77b6a5cd 2024-12-02T21:14:42,405 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for e42e4828e881f12c74fbedcdb6b00ea2 2024-12-02T21:14:42,405 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1048): stopping wal replay for 215753e705337ae11b1b38ad77b6a5cd 2024-12-02T21:14:42,405 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1060): Cleaning up temporary data for 215753e705337ae11b1b38ad77b6a5cd 2024-12-02T21:14:42,408 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1093): writing seq id for 215753e705337ae11b1b38ad77b6a5cd 2024-12-02T21:14:42,409 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/e42e4828e881f12c74fbedcdb6b00ea2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T21:14:42,409 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened e42e4828e881f12c74fbedcdb6b00ea2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68928472, jitterRate=0.027114272117614746}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T21:14:42,410 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e42e4828e881f12c74fbedcdb6b00ea2 2024-12-02T21:14:42,410 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for e42e4828e881f12c74fbedcdb6b00ea2: Running coprocessor pre-open hook at 1733174082392Writing region info on filesystem at 1733174082392Initializing all the Stores at 1733174082394 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174082394Cleaning up temporary data from old regions at 1733174082402 (+8 ms)Running coprocessor post-open hooks at 1733174082410 (+8 ms)Region opened successfully at 1733174082410 2024-12-02T21:14:42,412 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1733174081291.e42e4828e881f12c74fbedcdb6b00ea2., pid=40, masterSystemTime=1733174082358 2024-12-02T21:14:42,412 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/215753e705337ae11b1b38ad77b6a5cd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T21:14:42,416 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1114): Opened 215753e705337ae11b1b38ad77b6a5cd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60218890, jitterRate=-0.10266861319541931}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T21:14:42,417 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 215753e705337ae11b1b38ad77b6a5cd 2024-12-02T21:14:42,417 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1006): Region open journal for 215753e705337ae11b1b38ad77b6a5cd: Running coprocessor pre-open hook at 1733174082390Writing region info on filesystem at 1733174082390Initializing all the Stores at 1733174082394 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174082394Cleaning up temporary data from old regions at 1733174082405 (+11 ms)Running coprocessor post-open hooks at 1733174082417 (+12 ms)Region opened successfully at 1733174082417 2024-12-02T21:14:42,417 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1733174081291.e42e4828e881f12c74fbedcdb6b00ea2. 2024-12-02T21:14:42,417 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened test,sss,1733174081291.e42e4828e881f12c74fbedcdb6b00ea2. 2024-12-02T21:14:42,418 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open test,mmm,1733174081291.292fd0b4e0f07516cc25c069181e3111. 2024-12-02T21:14:42,418 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => 292fd0b4e0f07516cc25c069181e3111, NAME => 'test,mmm,1733174081291.292fd0b4e0f07516cc25c069181e3111.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-12-02T21:14:42,418 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 292fd0b4e0f07516cc25c069181e3111 2024-12-02T21:14:42,418 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated test,mmm,1733174081291.292fd0b4e0f07516cc25c069181e3111.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:14:42,419 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for 292fd0b4e0f07516cc25c069181e3111 2024-12-02T21:14:42,419 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for 292fd0b4e0f07516cc25c069181e3111 2024-12-02T21:14:42,421 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1733174081291.215753e705337ae11b1b38ad77b6a5cd., pid=42, masterSystemTime=1733174082379 2024-12-02T21:14:42,421 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=e42e4828e881f12c74fbedcdb6b00ea2, regionState=OPEN, openSeqNum=2, regionLocation=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:42,423 INFO [StoreOpener-292fd0b4e0f07516cc25c069181e3111-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 292fd0b4e0f07516cc25c069181e3111 2024-12-02T21:14:42,425 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1733174081291.215753e705337ae11b1b38ad77b6a5cd. 2024-12-02T21:14:42,425 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(153): Opened test,rrr,1733174081291.215753e705337ae11b1b38ad77b6a5cd. 2024-12-02T21:14:42,426 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(132): Open test,fff,1733174081291.e87d99b937228303e28b7ea0373ddc70. 2024-12-02T21:14:42,426 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7752): Opening region: {ENCODED => e87d99b937228303e28b7ea0373ddc70, NAME => 'test,fff,1733174081291.e87d99b937228303e28b7ea0373ddc70.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-12-02T21:14:42,426 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test e87d99b937228303e28b7ea0373ddc70 2024-12-02T21:14:42,426 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(898): Instantiated test,fff,1733174081291.e87d99b937228303e28b7ea0373ddc70.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:14:42,427 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7794): checking encryption for e87d99b937228303e28b7ea0373ddc70 2024-12-02T21:14:42,427 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7797): checking classloading for e87d99b937228303e28b7ea0373ddc70 2024-12-02T21:14:42,429 INFO [StoreOpener-292fd0b4e0f07516cc25c069181e3111-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 292fd0b4e0f07516cc25c069181e3111 columnFamilyName family 2024-12-02T21:14:42,429 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure e42e4828e881f12c74fbedcdb6b00ea2, server=87c3fdb6c570,40413,1733174078061 because future has completed 2024-12-02T21:14:42,429 DEBUG [StoreOpener-292fd0b4e0f07516cc25c069181e3111-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:14:42,430 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=215753e705337ae11b1b38ad77b6a5cd, regionState=OPEN, openSeqNum=2, regionLocation=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:42,430 INFO [StoreOpener-292fd0b4e0f07516cc25c069181e3111-1 {}] regionserver.HStore(327): Store=292fd0b4e0f07516cc25c069181e3111/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T21:14:42,430 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for 292fd0b4e0f07516cc25c069181e3111 2024-12-02T21:14:42,432 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/292fd0b4e0f07516cc25c069181e3111 2024-12-02T21:14:42,434 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 215753e705337ae11b1b38ad77b6a5cd, server=87c3fdb6c570,36585,1733174078182 because future has completed 2024-12-02T21:14:42,433 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/292fd0b4e0f07516cc25c069181e3111 2024-12-02T21:14:42,435 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for 292fd0b4e0f07516cc25c069181e3111 2024-12-02T21:14:42,435 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for 292fd0b4e0f07516cc25c069181e3111 2024-12-02T21:14:42,439 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for 292fd0b4e0f07516cc25c069181e3111 2024-12-02T21:14:42,440 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=23 2024-12-02T21:14:42,440 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure e42e4828e881f12c74fbedcdb6b00ea2, server=87c3fdb6c570,40413,1733174078061 in 182 msec 2024-12-02T21:14:42,444 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e42e4828e881f12c74fbedcdb6b00ea2, ASSIGN in 407 msec 2024-12-02T21:14:42,444 INFO [StoreOpener-e87d99b937228303e28b7ea0373ddc70-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region e87d99b937228303e28b7ea0373ddc70 2024-12-02T21:14:42,445 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open test,xxx,1733174081291.06d131e39cb4f5b02db4ec472d438a26. 2024-12-02T21:14:42,446 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => 06d131e39cb4f5b02db4ec472d438a26, NAME => 'test,xxx,1733174081291.06d131e39cb4f5b02db4ec472d438a26.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-12-02T21:14:42,446 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 06d131e39cb4f5b02db4ec472d438a26 2024-12-02T21:14:42,446 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated test,xxx,1733174081291.06d131e39cb4f5b02db4ec472d438a26.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:14:42,447 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for 06d131e39cb4f5b02db4ec472d438a26 2024-12-02T21:14:42,447 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for 06d131e39cb4f5b02db4ec472d438a26 2024-12-02T21:14:42,452 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=22 2024-12-02T21:14:42,452 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure 215753e705337ae11b1b38ad77b6a5cd, server=87c3fdb6c570,36585,1733174078182 in 181 msec 2024-12-02T21:14:42,453 INFO [StoreOpener-e87d99b937228303e28b7ea0373ddc70-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e87d99b937228303e28b7ea0373ddc70 columnFamilyName family 2024-12-02T21:14:42,453 DEBUG [StoreOpener-e87d99b937228303e28b7ea0373ddc70-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:14:42,454 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/292fd0b4e0f07516cc25c069181e3111/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T21:14:42,455 INFO [StoreOpener-e87d99b937228303e28b7ea0373ddc70-1 {}] regionserver.HStore(327): Store=e87d99b937228303e28b7ea0373ddc70/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T21:14:42,455 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1038): replaying wal for e87d99b937228303e28b7ea0373ddc70 2024-12-02T21:14:42,455 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened 292fd0b4e0f07516cc25c069181e3111; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69614920, jitterRate=0.03734314441680908}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T21:14:42,455 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 292fd0b4e0f07516cc25c069181e3111 2024-12-02T21:14:42,455 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for 292fd0b4e0f07516cc25c069181e3111: Running coprocessor pre-open hook at 1733174082419Writing region info on filesystem at 1733174082419Initializing all the Stores at 1733174082421 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174082421Cleaning up temporary data from old regions at 1733174082435 (+14 ms)Running coprocessor post-open hooks at 1733174082455 (+20 ms)Region opened successfully at 1733174082455 2024-12-02T21:14:42,456 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/e87d99b937228303e28b7ea0373ddc70 2024-12-02T21:14:42,456 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1733174081291.292fd0b4e0f07516cc25c069181e3111., pid=38, masterSystemTime=1733174082358 2024-12-02T21:14:42,457 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=215753e705337ae11b1b38ad77b6a5cd, ASSIGN in 420 msec 2024-12-02T21:14:42,457 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/e87d99b937228303e28b7ea0373ddc70 2024-12-02T21:14:42,458 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1048): stopping wal replay for e87d99b937228303e28b7ea0373ddc70 2024-12-02T21:14:42,458 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1060): Cleaning up temporary data for e87d99b937228303e28b7ea0373ddc70 2024-12-02T21:14:42,459 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1733174081291.292fd0b4e0f07516cc25c069181e3111. 2024-12-02T21:14:42,460 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened test,mmm,1733174081291.292fd0b4e0f07516cc25c069181e3111. 2024-12-02T21:14:42,460 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open test,ddd,1733174081291.ff611163b4ab91c50d940651a0065c34. 2024-12-02T21:14:42,460 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => ff611163b4ab91c50d940651a0065c34, NAME => 'test,ddd,1733174081291.ff611163b4ab91c50d940651a0065c34.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-12-02T21:14:42,460 INFO [StoreOpener-06d131e39cb4f5b02db4ec472d438a26-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 06d131e39cb4f5b02db4ec472d438a26 2024-12-02T21:14:42,460 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ff611163b4ab91c50d940651a0065c34 2024-12-02T21:14:42,460 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated test,ddd,1733174081291.ff611163b4ab91c50d940651a0065c34.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:14:42,461 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=292fd0b4e0f07516cc25c069181e3111, regionState=OPEN, openSeqNum=2, regionLocation=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:42,461 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for ff611163b4ab91c50d940651a0065c34 2024-12-02T21:14:42,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T21:14:42,461 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for ff611163b4ab91c50d940651a0065c34 2024-12-02T21:14:42,462 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1093): writing seq id for e87d99b937228303e28b7ea0373ddc70 2024-12-02T21:14:42,464 INFO [StoreOpener-06d131e39cb4f5b02db4ec472d438a26-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06d131e39cb4f5b02db4ec472d438a26 columnFamilyName family 2024-12-02T21:14:42,464 INFO [StoreOpener-ff611163b4ab91c50d940651a0065c34-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ff611163b4ab91c50d940651a0065c34 2024-12-02T21:14:42,464 DEBUG [StoreOpener-06d131e39cb4f5b02db4ec472d438a26-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:14:42,465 INFO [StoreOpener-06d131e39cb4f5b02db4ec472d438a26-1 {}] regionserver.HStore(327): Store=06d131e39cb4f5b02db4ec472d438a26/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T21:14:42,466 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 292fd0b4e0f07516cc25c069181e3111, server=87c3fdb6c570,40413,1733174078061 because future has completed 2024-12-02T21:14:42,466 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for 06d131e39cb4f5b02db4ec472d438a26 2024-12-02T21:14:42,468 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/06d131e39cb4f5b02db4ec472d438a26 2024-12-02T21:14:42,468 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/06d131e39cb4f5b02db4ec472d438a26 2024-12-02T21:14:42,469 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for 06d131e39cb4f5b02db4ec472d438a26 2024-12-02T21:14:42,470 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for 06d131e39cb4f5b02db4ec472d438a26 2024-12-02T21:14:42,472 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=17 2024-12-02T21:14:42,472 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure 292fd0b4e0f07516cc25c069181e3111, server=87c3fdb6c570,40413,1733174078061 in 223 msec 2024-12-02T21:14:42,473 INFO [StoreOpener-ff611163b4ab91c50d940651a0065c34-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ff611163b4ab91c50d940651a0065c34 columnFamilyName family 2024-12-02T21:14:42,473 DEBUG [StoreOpener-ff611163b4ab91c50d940651a0065c34-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:14:42,474 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for 06d131e39cb4f5b02db4ec472d438a26 2024-12-02T21:14:42,474 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/e87d99b937228303e28b7ea0373ddc70/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T21:14:42,475 INFO [StoreOpener-ff611163b4ab91c50d940651a0065c34-1 {}] regionserver.HStore(327): Store=ff611163b4ab91c50d940651a0065c34/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T21:14:42,475 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for ff611163b4ab91c50d940651a0065c34 2024-12-02T21:14:42,476 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1114): Opened e87d99b937228303e28b7ea0373ddc70; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74243179, jitterRate=0.10630957782268524}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T21:14:42,476 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e87d99b937228303e28b7ea0373ddc70 2024-12-02T21:14:42,476 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1006): Region open journal for e87d99b937228303e28b7ea0373ddc70: Running coprocessor pre-open hook at 1733174082427Writing region info on filesystem at 1733174082427Initializing all the Stores at 1733174082429 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174082429Cleaning up temporary data from old regions at 1733174082458 (+29 ms)Running coprocessor post-open hooks at 1733174082476 (+18 ms)Region opened successfully at 1733174082476 2024-12-02T21:14:42,477 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/ff611163b4ab91c50d940651a0065c34 2024-12-02T21:14:42,477 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1733174081291.e87d99b937228303e28b7ea0373ddc70., pid=44, masterSystemTime=1733174082379 2024-12-02T21:14:42,479 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/ff611163b4ab91c50d940651a0065c34 2024-12-02T21:14:42,479 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for ff611163b4ab91c50d940651a0065c34 2024-12-02T21:14:42,480 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for ff611163b4ab91c50d940651a0065c34 2024-12-02T21:14:42,480 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/06d131e39cb4f5b02db4ec472d438a26/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T21:14:42,481 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=292fd0b4e0f07516cc25c069181e3111, ASSIGN in 439 msec 2024-12-02T21:14:42,482 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1733174081291.e87d99b937228303e28b7ea0373ddc70. 2024-12-02T21:14:42,482 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(153): Opened test,fff,1733174081291.e87d99b937228303e28b7ea0373ddc70. 2024-12-02T21:14:42,483 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for ff611163b4ab91c50d940651a0065c34 2024-12-02T21:14:42,482 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened 06d131e39cb4f5b02db4ec472d438a26; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67131726, jitterRate=3.406703472137451E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T21:14:42,483 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 06d131e39cb4f5b02db4ec472d438a26 2024-12-02T21:14:42,483 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open test,,1733174081291.85ca3a9a09d75c0941c58f5b83584a8e. 2024-12-02T21:14:42,483 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for 06d131e39cb4f5b02db4ec472d438a26: Running coprocessor pre-open hook at 1733174082447Writing region info on filesystem at 1733174082447Initializing all the Stores at 1733174082449 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174082449Cleaning up temporary data from old regions at 1733174082470 (+21 ms)Running coprocessor post-open hooks at 1733174082483 (+13 ms)Region opened successfully at 1733174082483 2024-12-02T21:14:42,483 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => 85ca3a9a09d75c0941c58f5b83584a8e, NAME => 'test,,1733174081291.85ca3a9a09d75c0941c58f5b83584a8e.', STARTKEY => '', ENDKEY => 'bbb'} 2024-12-02T21:14:42,485 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 85ca3a9a09d75c0941c58f5b83584a8e 2024-12-02T21:14:42,485 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=e87d99b937228303e28b7ea0373ddc70, regionState=OPEN, openSeqNum=2, regionLocation=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:42,485 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated test,,1733174081291.85ca3a9a09d75c0941c58f5b83584a8e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:14:42,485 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for 85ca3a9a09d75c0941c58f5b83584a8e 2024-12-02T21:14:42,485 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for 85ca3a9a09d75c0941c58f5b83584a8e 2024-12-02T21:14:42,486 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1733174081291.06d131e39cb4f5b02db4ec472d438a26., pid=45, masterSystemTime=1733174082438 2024-12-02T21:14:42,489 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/ff611163b4ab91c50d940651a0065c34/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T21:14:42,490 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1733174081291.06d131e39cb4f5b02db4ec472d438a26. 2024-12-02T21:14:42,490 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened test,xxx,1733174081291.06d131e39cb4f5b02db4ec472d438a26. 2024-12-02T21:14:42,490 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(132): Open test,lll,1733174081291.0d0be65f6ffd7ebb4c8b2fee37042136. 2024-12-02T21:14:42,490 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7752): Opening region: {ENCODED => 0d0be65f6ffd7ebb4c8b2fee37042136, NAME => 'test,lll,1733174081291.0d0be65f6ffd7ebb4c8b2fee37042136.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-12-02T21:14:42,490 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 0d0be65f6ffd7ebb4c8b2fee37042136 2024-12-02T21:14:42,490 INFO [StoreOpener-85ca3a9a09d75c0941c58f5b83584a8e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 85ca3a9a09d75c0941c58f5b83584a8e 2024-12-02T21:14:42,490 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened ff611163b4ab91c50d940651a0065c34; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62323947, jitterRate=-0.07130081951618195}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T21:14:42,491 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(898): Instantiated test,lll,1733174081291.0d0be65f6ffd7ebb4c8b2fee37042136.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:14:42,491 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ff611163b4ab91c50d940651a0065c34 2024-12-02T21:14:42,491 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7794): checking encryption for 0d0be65f6ffd7ebb4c8b2fee37042136 2024-12-02T21:14:42,491 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7797): checking classloading for 0d0be65f6ffd7ebb4c8b2fee37042136 2024-12-02T21:14:42,491 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for ff611163b4ab91c50d940651a0065c34: Running coprocessor pre-open hook at 1733174082461Writing region info on filesystem at 1733174082461Initializing all the Stores at 1733174082462 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174082462Cleaning up temporary data from old regions at 1733174082480 (+18 ms)Running coprocessor post-open hooks at 1733174082491 (+11 ms)Region opened successfully at 1733174082491 2024-12-02T21:14:42,493 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ddd,1733174081291.ff611163b4ab91c50d940651a0065c34., pid=36, masterSystemTime=1733174082358 2024-12-02T21:14:42,493 INFO [StoreOpener-85ca3a9a09d75c0941c58f5b83584a8e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 85ca3a9a09d75c0941c58f5b83584a8e columnFamilyName family 2024-12-02T21:14:42,493 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=06d131e39cb4f5b02db4ec472d438a26, regionState=OPEN, openSeqNum=2, regionLocation=87c3fdb6c570,33459,1733174078298 2024-12-02T21:14:42,493 DEBUG [StoreOpener-85ca3a9a09d75c0941c58f5b83584a8e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:14:42,494 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure e87d99b937228303e28b7ea0373ddc70, server=87c3fdb6c570,36585,1733174078182 because future has completed 2024-12-02T21:14:42,494 INFO [StoreOpener-85ca3a9a09d75c0941c58f5b83584a8e-1 {}] regionserver.HStore(327): Store=85ca3a9a09d75c0941c58f5b83584a8e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T21:14:42,495 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for 85ca3a9a09d75c0941c58f5b83584a8e 2024-12-02T21:14:42,497 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/85ca3a9a09d75c0941c58f5b83584a8e 2024-12-02T21:14:42,498 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06d131e39cb4f5b02db4ec472d438a26, server=87c3fdb6c570,33459,1733174078298 because future has completed 2024-12-02T21:14:42,498 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ddd,1733174081291.ff611163b4ab91c50d940651a0065c34. 2024-12-02T21:14:42,498 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/85ca3a9a09d75c0941c58f5b83584a8e 2024-12-02T21:14:42,498 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened test,ddd,1733174081291.ff611163b4ab91c50d940651a0065c34. 2024-12-02T21:14:42,499 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open test,vvv,1733174081291.6a2fb6e9d39bf04774472d3b4c70810a. 2024-12-02T21:14:42,499 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42045 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=87c3fdb6c570,33459,1733174078298, table=test, region=06d131e39cb4f5b02db4ec472d438a26. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-02T21:14:42,499 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => 6a2fb6e9d39bf04774472d3b4c70810a, NAME => 'test,vvv,1733174081291.6a2fb6e9d39bf04774472d3b4c70810a.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-12-02T21:14:42,499 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for 85ca3a9a09d75c0941c58f5b83584a8e 2024-12-02T21:14:42,499 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for 85ca3a9a09d75c0941c58f5b83584a8e 2024-12-02T21:14:42,499 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 6a2fb6e9d39bf04774472d3b4c70810a 2024-12-02T21:14:42,500 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated test,vvv,1733174081291.6a2fb6e9d39bf04774472d3b4c70810a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:14:42,500 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for 6a2fb6e9d39bf04774472d3b4c70810a 2024-12-02T21:14:42,500 INFO [StoreOpener-0d0be65f6ffd7ebb4c8b2fee37042136-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 0d0be65f6ffd7ebb4c8b2fee37042136 2024-12-02T21:14:42,501 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for 6a2fb6e9d39bf04774472d3b4c70810a 2024-12-02T21:14:42,502 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for 85ca3a9a09d75c0941c58f5b83584a8e 2024-12-02T21:14:42,504 INFO [StoreOpener-6a2fb6e9d39bf04774472d3b4c70810a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 6a2fb6e9d39bf04774472d3b4c70810a 2024-12-02T21:14:42,505 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=28 2024-12-02T21:14:42,505 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=28, state=SUCCESS, hasLock=false; OpenRegionProcedure 06d131e39cb4f5b02db4ec472d438a26, server=87c3fdb6c570,33459,1733174078298 in 217 msec 2024-12-02T21:14:42,506 INFO [StoreOpener-0d0be65f6ffd7ebb4c8b2fee37042136-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d0be65f6ffd7ebb4c8b2fee37042136 columnFamilyName family 2024-12-02T21:14:42,506 INFO [StoreOpener-6a2fb6e9d39bf04774472d3b4c70810a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6a2fb6e9d39bf04774472d3b4c70810a columnFamilyName family 2024-12-02T21:14:42,506 DEBUG [StoreOpener-0d0be65f6ffd7ebb4c8b2fee37042136-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:14:42,506 DEBUG [StoreOpener-6a2fb6e9d39bf04774472d3b4c70810a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:14:42,507 INFO [StoreOpener-6a2fb6e9d39bf04774472d3b4c70810a-1 {}] regionserver.HStore(327): Store=6a2fb6e9d39bf04774472d3b4c70810a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T21:14:42,507 INFO [StoreOpener-0d0be65f6ffd7ebb4c8b2fee37042136-1 {}] regionserver.HStore(327): Store=0d0be65f6ffd7ebb4c8b2fee37042136/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T21:14:42,507 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for 6a2fb6e9d39bf04774472d3b4c70810a 2024-12-02T21:14:42,507 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1038): replaying wal for 0d0be65f6ffd7ebb4c8b2fee37042136 2024-12-02T21:14:42,508 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/85ca3a9a09d75c0941c58f5b83584a8e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T21:14:42,509 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/6a2fb6e9d39bf04774472d3b4c70810a 2024-12-02T21:14:42,509 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN in 472 msec 2024-12-02T21:14:42,509 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/6a2fb6e9d39bf04774472d3b4c70810a 2024-12-02T21:14:42,509 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened 85ca3a9a09d75c0941c58f5b83584a8e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69041578, jitterRate=0.02879968285560608}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T21:14:42,509 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 85ca3a9a09d75c0941c58f5b83584a8e 2024-12-02T21:14:42,510 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for 85ca3a9a09d75c0941c58f5b83584a8e: Running coprocessor pre-open hook at 1733174082485Writing region info on filesystem at 1733174082485Initializing all the Stores at 1733174082487 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174082487Cleaning up temporary data from old regions at 1733174082499 (+12 ms)Running coprocessor post-open hooks at 1733174082510 (+11 ms)Region opened successfully at 1733174082510 2024-12-02T21:14:42,510 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for 6a2fb6e9d39bf04774472d3b4c70810a 2024-12-02T21:14:42,510 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for 6a2fb6e9d39bf04774472d3b4c70810a 2024-12-02T21:14:42,510 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/0d0be65f6ffd7ebb4c8b2fee37042136 2024-12-02T21:14:42,511 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/0d0be65f6ffd7ebb4c8b2fee37042136 2024-12-02T21:14:42,512 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1048): stopping wal replay for 0d0be65f6ffd7ebb4c8b2fee37042136 2024-12-02T21:14:42,512 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=ff611163b4ab91c50d940651a0065c34, regionState=OPEN, openSeqNum=2, regionLocation=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:42,512 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1060): Cleaning up temporary data for 0d0be65f6ffd7ebb4c8b2fee37042136 2024-12-02T21:14:42,513 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1733174081291.85ca3a9a09d75c0941c58f5b83584a8e., pid=46, masterSystemTime=1733174082379 2024-12-02T21:14:42,514 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for 6a2fb6e9d39bf04774472d3b4c70810a 2024-12-02T21:14:42,517 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure ff611163b4ab91c50d940651a0065c34, server=87c3fdb6c570,40413,1733174078061 because future has completed 2024-12-02T21:14:42,519 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for test,,1733174081291.85ca3a9a09d75c0941c58f5b83584a8e. 2024-12-02T21:14:42,519 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened test,,1733174081291.85ca3a9a09d75c0941c58f5b83584a8e. 2024-12-02T21:14:42,520 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open test,jjj,1733174081291.5d60c566b39516dd875aaf2bd7def402. 2024-12-02T21:14:42,520 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => 5d60c566b39516dd875aaf2bd7def402, NAME => 'test,jjj,1733174081291.5d60c566b39516dd875aaf2bd7def402.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-12-02T21:14:42,520 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): writing seq id for 0d0be65f6ffd7ebb4c8b2fee37042136 2024-12-02T21:14:42,520 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 5d60c566b39516dd875aaf2bd7def402 2024-12-02T21:14:42,520 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated test,jjj,1733174081291.5d60c566b39516dd875aaf2bd7def402.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:14:42,521 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for 5d60c566b39516dd875aaf2bd7def402 2024-12-02T21:14:42,521 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for 5d60c566b39516dd875aaf2bd7def402 2024-12-02T21:14:42,525 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/6a2fb6e9d39bf04774472d3b4c70810a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T21:14:42,526 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened 6a2fb6e9d39bf04774472d3b4c70810a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73949529, jitterRate=0.10193385183811188}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T21:14:42,526 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6a2fb6e9d39bf04774472d3b4c70810a 2024-12-02T21:14:42,526 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for 6a2fb6e9d39bf04774472d3b4c70810a: Running coprocessor pre-open hook at 1733174082502Writing region info on filesystem at 1733174082502Initializing all the Stores at 1733174082503 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174082503Cleaning up temporary data from old regions at 1733174082510 (+7 ms)Running coprocessor post-open hooks at 1733174082526 (+16 ms)Region opened successfully at 1733174082526 2024-12-02T21:14:42,527 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1733174081291.6a2fb6e9d39bf04774472d3b4c70810a., pid=41, masterSystemTime=1733174082358 2024-12-02T21:14:42,527 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=10 2024-12-02T21:14:42,528 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure e87d99b937228303e28b7ea0373ddc70, server=87c3fdb6c570,36585,1733174078182 in 246 msec 2024-12-02T21:14:42,528 INFO [StoreOpener-5d60c566b39516dd875aaf2bd7def402-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 5d60c566b39516dd875aaf2bd7def402 2024-12-02T21:14:42,531 INFO [StoreOpener-5d60c566b39516dd875aaf2bd7def402-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5d60c566b39516dd875aaf2bd7def402 columnFamilyName family 2024-12-02T21:14:42,531 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1733174081291.6a2fb6e9d39bf04774472d3b4c70810a. 2024-12-02T21:14:42,531 DEBUG [StoreOpener-5d60c566b39516dd875aaf2bd7def402-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:14:42,531 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened test,vvv,1733174081291.6a2fb6e9d39bf04774472d3b4c70810a. 2024-12-02T21:14:42,531 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e87d99b937228303e28b7ea0373ddc70, ASSIGN in 495 msec 2024-12-02T21:14:42,531 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open test,ppp,1733174081291.20545c8ee2d66091a91ff9697d1f18ca. 2024-12-02T21:14:42,531 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => 20545c8ee2d66091a91ff9697d1f18ca, NAME => 'test,ppp,1733174081291.20545c8ee2d66091a91ff9697d1f18ca.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-12-02T21:14:42,532 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 20545c8ee2d66091a91ff9697d1f18ca 2024-12-02T21:14:42,532 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated test,ppp,1733174081291.20545c8ee2d66091a91ff9697d1f18ca.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:14:42,532 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for 20545c8ee2d66091a91ff9697d1f18ca 2024-12-02T21:14:42,532 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for 20545c8ee2d66091a91ff9697d1f18ca 2024-12-02T21:14:42,532 INFO [StoreOpener-5d60c566b39516dd875aaf2bd7def402-1 {}] regionserver.HStore(327): Store=5d60c566b39516dd875aaf2bd7def402/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T21:14:42,532 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for 5d60c566b39516dd875aaf2bd7def402 2024-12-02T21:14:42,533 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=6a2fb6e9d39bf04774472d3b4c70810a, regionState=OPEN, openSeqNum=2, regionLocation=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:42,533 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/5d60c566b39516dd875aaf2bd7def402 2024-12-02T21:14:42,534 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/5d60c566b39516dd875aaf2bd7def402 2024-12-02T21:14:42,535 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for 5d60c566b39516dd875aaf2bd7def402 2024-12-02T21:14:42,535 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for 5d60c566b39516dd875aaf2bd7def402 2024-12-02T21:14:42,537 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=85ca3a9a09d75c0941c58f5b83584a8e, regionState=OPEN, openSeqNum=2, regionLocation=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:42,537 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6a2fb6e9d39bf04774472d3b4c70810a, server=87c3fdb6c570,40413,1733174078061 because future has completed 2024-12-02T21:14:42,539 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for 5d60c566b39516dd875aaf2bd7def402 2024-12-02T21:14:42,540 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 85ca3a9a09d75c0941c58f5b83584a8e, server=87c3fdb6c570,36585,1733174078182 because future has completed 2024-12-02T21:14:42,542 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/0d0be65f6ffd7ebb4c8b2fee37042136/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T21:14:42,544 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=26 2024-12-02T21:14:42,545 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 6a2fb6e9d39bf04774472d3b4c70810a, server=87c3fdb6c570,40413,1733174078061 in 282 msec 2024-12-02T21:14:42,546 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1114): Opened 0d0be65f6ffd7ebb4c8b2fee37042136; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64921006, jitterRate=-0.032601624727249146}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T21:14:42,546 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0d0be65f6ffd7ebb4c8b2fee37042136 2024-12-02T21:14:42,546 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1006): Region open journal for 0d0be65f6ffd7ebb4c8b2fee37042136: Running coprocessor pre-open hook at 1733174082491Writing region info on filesystem at 1733174082491Initializing all the Stores at 1733174082494 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174082494Cleaning up temporary data from old regions at 1733174082512 (+18 ms)Running coprocessor post-open hooks at 1733174082546 (+34 ms)Region opened successfully at 1733174082546 2024-12-02T21:14:42,547 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1733174081291.0d0be65f6ffd7ebb4c8b2fee37042136., pid=50, masterSystemTime=1733174082438 2024-12-02T21:14:42,548 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6a2fb6e9d39bf04774472d3b4c70810a, ASSIGN in 512 msec 2024-12-02T21:14:42,548 INFO [StoreOpener-20545c8ee2d66091a91ff9697d1f18ca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 20545c8ee2d66091a91ff9697d1f18ca 2024-12-02T21:14:42,550 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/5d60c566b39516dd875aaf2bd7def402/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T21:14:42,551 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened 5d60c566b39516dd875aaf2bd7def402; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74180612, jitterRate=0.10537725687026978}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T21:14:42,551 INFO [StoreOpener-20545c8ee2d66091a91ff9697d1f18ca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 20545c8ee2d66091a91ff9697d1f18ca columnFamilyName family 2024-12-02T21:14:42,551 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5d60c566b39516dd875aaf2bd7def402 2024-12-02T21:14:42,551 DEBUG [StoreOpener-20545c8ee2d66091a91ff9697d1f18ca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:14:42,551 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for 5d60c566b39516dd875aaf2bd7def402: Running coprocessor pre-open hook at 1733174082521Writing region info on filesystem at 1733174082521Initializing all the Stores at 1733174082523 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174082523Cleaning up temporary data from old regions at 1733174082535 (+12 ms)Running coprocessor post-open hooks at 1733174082551 (+16 ms)Region opened successfully at 1733174082551 2024-12-02T21:14:42,551 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1733174081291.0d0be65f6ffd7ebb4c8b2fee37042136. 2024-12-02T21:14:42,551 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(153): Opened test,lll,1733174081291.0d0be65f6ffd7ebb4c8b2fee37042136. 2024-12-02T21:14:42,552 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(132): Open test,ttt,1733174081291.d63da23e8236e18b416716ec838dd43e. 2024-12-02T21:14:42,552 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7752): Opening region: {ENCODED => d63da23e8236e18b416716ec838dd43e, NAME => 'test,ttt,1733174081291.d63da23e8236e18b416716ec838dd43e.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-12-02T21:14:42,552 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d63da23e8236e18b416716ec838dd43e 2024-12-02T21:14:42,552 INFO [StoreOpener-20545c8ee2d66091a91ff9697d1f18ca-1 {}] regionserver.HStore(327): Store=20545c8ee2d66091a91ff9697d1f18ca/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T21:14:42,552 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(898): Instantiated test,ttt,1733174081291.d63da23e8236e18b416716ec838dd43e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:14:42,552 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for 20545c8ee2d66091a91ff9697d1f18ca 2024-12-02T21:14:42,552 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7794): checking encryption for d63da23e8236e18b416716ec838dd43e 2024-12-02T21:14:42,553 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7797): checking classloading for d63da23e8236e18b416716ec838dd43e 2024-12-02T21:14:42,553 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/20545c8ee2d66091a91ff9697d1f18ca 2024-12-02T21:14:42,554 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for test,jjj,1733174081291.5d60c566b39516dd875aaf2bd7def402., pid=43, masterSystemTime=1733174082379 2024-12-02T21:14:42,554 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/20545c8ee2d66091a91ff9697d1f18ca 2024-12-02T21:14:42,555 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for 20545c8ee2d66091a91ff9697d1f18ca 2024-12-02T21:14:42,555 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for 20545c8ee2d66091a91ff9697d1f18ca 2024-12-02T21:14:42,557 INFO [StoreOpener-d63da23e8236e18b416716ec838dd43e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d63da23e8236e18b416716ec838dd43e 2024-12-02T21:14:42,558 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for 20545c8ee2d66091a91ff9697d1f18ca 2024-12-02T21:14:42,559 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=5 2024-12-02T21:14:42,559 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 85ca3a9a09d75c0941c58f5b83584a8e, server=87c3fdb6c570,36585,1733174078182 in 257 msec 2024-12-02T21:14:42,559 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for test,jjj,1733174081291.5d60c566b39516dd875aaf2bd7def402. 2024-12-02T21:14:42,559 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened test,jjj,1733174081291.5d60c566b39516dd875aaf2bd7def402. 2024-12-02T21:14:42,559 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(132): Open test,ooo,1733174081291.4197cd4f7ea2f3d6256be6836e9834da. 2024-12-02T21:14:42,559 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7752): Opening region: {ENCODED => 4197cd4f7ea2f3d6256be6836e9834da, NAME => 'test,ooo,1733174081291.4197cd4f7ea2f3d6256be6836e9834da.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-12-02T21:14:42,560 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4197cd4f7ea2f3d6256be6836e9834da 2024-12-02T21:14:42,560 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(898): Instantiated test,ooo,1733174081291.4197cd4f7ea2f3d6256be6836e9834da.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:14:42,560 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7794): checking encryption for 4197cd4f7ea2f3d6256be6836e9834da 2024-12-02T21:14:42,560 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7797): checking classloading for 4197cd4f7ea2f3d6256be6836e9834da 2024-12-02T21:14:42,561 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=5d60c566b39516dd875aaf2bd7def402, regionState=OPEN, openSeqNum=2, regionLocation=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:42,563 INFO [StoreOpener-d63da23e8236e18b416716ec838dd43e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d63da23e8236e18b416716ec838dd43e columnFamilyName family 2024-12-02T21:14:42,563 DEBUG [StoreOpener-d63da23e8236e18b416716ec838dd43e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:14:42,564 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5d60c566b39516dd875aaf2bd7def402, server=87c3fdb6c570,36585,1733174078182 because future has completed 2024-12-02T21:14:42,564 INFO [StoreOpener-d63da23e8236e18b416716ec838dd43e-1 {}] regionserver.HStore(327): Store=d63da23e8236e18b416716ec838dd43e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T21:14:42,564 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1038): replaying wal for d63da23e8236e18b416716ec838dd43e 2024-12-02T21:14:42,566 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/d63da23e8236e18b416716ec838dd43e 2024-12-02T21:14:42,566 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/d63da23e8236e18b416716ec838dd43e 2024-12-02T21:14:42,567 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=85ca3a9a09d75c0941c58f5b83584a8e, ASSIGN in 531 msec 2024-12-02T21:14:42,567 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1048): stopping wal replay for d63da23e8236e18b416716ec838dd43e 2024-12-02T21:14:42,567 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1060): Cleaning up temporary data for d63da23e8236e18b416716ec838dd43e 2024-12-02T21:14:42,570 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1093): writing seq id for d63da23e8236e18b416716ec838dd43e 2024-12-02T21:14:42,572 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=14 2024-12-02T21:14:42,572 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 5d60c566b39516dd875aaf2bd7def402, server=87c3fdb6c570,36585,1733174078182 in 297 msec 2024-12-02T21:14:42,575 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=5d60c566b39516dd875aaf2bd7def402, ASSIGN in 540 msec 2024-12-02T21:14:42,577 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=0d0be65f6ffd7ebb4c8b2fee37042136, regionState=OPEN, openSeqNum=2, regionLocation=87c3fdb6c570,33459,1733174078298 2024-12-02T21:14:42,578 INFO [StoreOpener-4197cd4f7ea2f3d6256be6836e9834da-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4197cd4f7ea2f3d6256be6836e9834da 2024-12-02T21:14:42,581 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0d0be65f6ffd7ebb4c8b2fee37042136, server=87c3fdb6c570,33459,1733174078298 because future has completed 2024-12-02T21:14:42,581 INFO [StoreOpener-4197cd4f7ea2f3d6256be6836e9834da-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4197cd4f7ea2f3d6256be6836e9834da columnFamilyName family 2024-12-02T21:14:42,581 DEBUG [StoreOpener-4197cd4f7ea2f3d6256be6836e9834da-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:14:42,582 INFO [StoreOpener-4197cd4f7ea2f3d6256be6836e9834da-1 {}] regionserver.HStore(327): Store=4197cd4f7ea2f3d6256be6836e9834da/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T21:14:42,583 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1038): replaying wal for 4197cd4f7ea2f3d6256be6836e9834da 2024-12-02T21:14:42,584 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/20545c8ee2d66091a91ff9697d1f18ca/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T21:14:42,584 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/4197cd4f7ea2f3d6256be6836e9834da 2024-12-02T21:14:42,585 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened 20545c8ee2d66091a91ff9697d1f18ca; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62851481, jitterRate=-0.06343995034694672}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T21:14:42,585 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 20545c8ee2d66091a91ff9697d1f18ca 2024-12-02T21:14:42,585 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for 20545c8ee2d66091a91ff9697d1f18ca: Running coprocessor pre-open hook at 1733174082532Writing region info on filesystem at 1733174082532Initializing all the Stores at 1733174082534 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174082534Cleaning up temporary data from old regions at 1733174082555 (+21 ms)Running coprocessor post-open hooks at 1733174082585 (+30 ms)Region opened successfully at 1733174082585 2024-12-02T21:14:42,585 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/4197cd4f7ea2f3d6256be6836e9834da 2024-12-02T21:14:42,587 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1048): stopping wal replay for 4197cd4f7ea2f3d6256be6836e9834da 2024-12-02T21:14:42,587 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1060): Cleaning up temporary data for 4197cd4f7ea2f3d6256be6836e9834da 2024-12-02T21:14:42,588 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42045 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=87c3fdb6c570,40413,1733174078061, table=test, region=ff611163b4ab91c50d940651a0065c34. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-02T21:14:42,590 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1733174081291.20545c8ee2d66091a91ff9697d1f18ca., pid=39, masterSystemTime=1733174082358 2024-12-02T21:14:42,597 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ppp,1733174081291.20545c8ee2d66091a91ff9697d1f18ca. 2024-12-02T21:14:42,597 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened test,ppp,1733174081291.20545c8ee2d66091a91ff9697d1f18ca. 2024-12-02T21:14:42,597 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open test,zzz,1733174081291.a36d8f99c2aef77f88827585926ad989. 2024-12-02T21:14:42,597 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => a36d8f99c2aef77f88827585926ad989, NAME => 'test,zzz,1733174081291.a36d8f99c2aef77f88827585926ad989.', STARTKEY => 'zzz', ENDKEY => ''} 2024-12-02T21:14:42,598 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test a36d8f99c2aef77f88827585926ad989 2024-12-02T21:14:42,598 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated test,zzz,1733174081291.a36d8f99c2aef77f88827585926ad989.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:14:42,599 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for a36d8f99c2aef77f88827585926ad989 2024-12-02T21:14:42,599 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for a36d8f99c2aef77f88827585926ad989 2024-12-02T21:14:42,602 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42045 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=87c3fdb6c570,33459,1733174078298, table=test, region=0d0be65f6ffd7ebb4c8b2fee37042136. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-02T21:14:42,604 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=8 2024-12-02T21:14:42,604 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure ff611163b4ab91c50d940651a0065c34, server=87c3fdb6c570,40413,1733174078061 in 361 msec 2024-12-02T21:14:42,604 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/d63da23e8236e18b416716ec838dd43e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T21:14:42,604 INFO [StoreOpener-a36d8f99c2aef77f88827585926ad989-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region a36d8f99c2aef77f88827585926ad989 2024-12-02T21:14:42,605 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1114): Opened d63da23e8236e18b416716ec838dd43e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69783769, jitterRate=0.039859190583229065}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T21:14:42,605 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d63da23e8236e18b416716ec838dd43e 2024-12-02T21:14:42,605 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1006): Region open journal for d63da23e8236e18b416716ec838dd43e: Running coprocessor pre-open hook at 1733174082553Writing region info on filesystem at 1733174082553Initializing all the Stores at 1733174082555 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174082555Cleaning up temporary data from old regions at 1733174082567 (+12 ms)Running coprocessor post-open hooks at 1733174082605 (+38 ms)Region opened successfully at 1733174082605 2024-12-02T21:14:42,607 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1733174081291.d63da23e8236e18b416716ec838dd43e., pid=54, masterSystemTime=1733174082438 2024-12-02T21:14:42,609 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1093): writing seq id for 4197cd4f7ea2f3d6256be6836e9834da 2024-12-02T21:14:42,609 INFO [StoreOpener-a36d8f99c2aef77f88827585926ad989-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a36d8f99c2aef77f88827585926ad989 columnFamilyName family 2024-12-02T21:14:42,609 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=ff611163b4ab91c50d940651a0065c34, ASSIGN in 571 msec 2024-12-02T21:14:42,609 DEBUG [StoreOpener-a36d8f99c2aef77f88827585926ad989-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:14:42,610 INFO [StoreOpener-a36d8f99c2aef77f88827585926ad989-1 {}] regionserver.HStore(327): Store=a36d8f99c2aef77f88827585926ad989/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T21:14:42,611 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for a36d8f99c2aef77f88827585926ad989 2024-12-02T21:14:42,612 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/a36d8f99c2aef77f88827585926ad989 2024-12-02T21:14:42,612 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=20545c8ee2d66091a91ff9697d1f18ca, regionState=OPEN, openSeqNum=2, regionLocation=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:42,614 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/4197cd4f7ea2f3d6256be6836e9834da/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T21:14:42,614 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/a36d8f99c2aef77f88827585926ad989 2024-12-02T21:14:42,614 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1733174081291.d63da23e8236e18b416716ec838dd43e. 2024-12-02T21:14:42,614 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(153): Opened test,ttt,1733174081291.d63da23e8236e18b416716ec838dd43e. 2024-12-02T21:14:42,615 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(132): Open test,www,1733174081291.1a4ea0dec946dcfe96ad162d5bcf5949. 2024-12-02T21:14:42,615 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7752): Opening region: {ENCODED => 1a4ea0dec946dcfe96ad162d5bcf5949, NAME => 'test,www,1733174081291.1a4ea0dec946dcfe96ad162d5bcf5949.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-12-02T21:14:42,615 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1114): Opened 4197cd4f7ea2f3d6256be6836e9834da; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66038696, jitterRate=-0.01594674587249756}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T21:14:42,615 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4197cd4f7ea2f3d6256be6836e9834da 2024-12-02T21:14:42,615 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 1a4ea0dec946dcfe96ad162d5bcf5949 2024-12-02T21:14:42,615 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(898): Instantiated test,www,1733174081291.1a4ea0dec946dcfe96ad162d5bcf5949.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:14:42,615 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1006): Region open journal for 4197cd4f7ea2f3d6256be6836e9834da: Running coprocessor pre-open hook at 1733174082561Writing region info on filesystem at 1733174082561Initializing all the Stores at 1733174082562 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174082562Cleaning up temporary data from old regions at 1733174082587 (+25 ms)Running coprocessor post-open hooks at 1733174082615 (+28 ms)Region opened successfully at 1733174082615 2024-12-02T21:14:42,615 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7794): checking encryption for 1a4ea0dec946dcfe96ad162d5bcf5949 2024-12-02T21:14:42,615 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7797): checking classloading for 1a4ea0dec946dcfe96ad162d5bcf5949 2024-12-02T21:14:42,616 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for a36d8f99c2aef77f88827585926ad989 2024-12-02T21:14:42,616 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for a36d8f99c2aef77f88827585926ad989 2024-12-02T21:14:42,620 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for a36d8f99c2aef77f88827585926ad989 2024-12-02T21:14:42,621 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 20545c8ee2d66091a91ff9697d1f18ca, server=87c3fdb6c570,40413,1733174078061 because future has completed 2024-12-02T21:14:42,622 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=d63da23e8236e18b416716ec838dd43e, regionState=OPEN, openSeqNum=2, regionLocation=87c3fdb6c570,33459,1733174078298 2024-12-02T21:14:42,624 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/a36d8f99c2aef77f88827585926ad989/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T21:14:42,624 INFO [StoreOpener-1a4ea0dec946dcfe96ad162d5bcf5949-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 1a4ea0dec946dcfe96ad162d5bcf5949 2024-12-02T21:14:42,625 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened a36d8f99c2aef77f88827585926ad989; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62543756, jitterRate=-0.06802541017532349}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T21:14:42,625 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a36d8f99c2aef77f88827585926ad989 2024-12-02T21:14:42,625 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for a36d8f99c2aef77f88827585926ad989: Running coprocessor pre-open hook at 1733174082599Writing region info on filesystem at 1733174082599Initializing all the Stores at 1733174082601 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174082601Cleaning up temporary data from old regions at 1733174082616 (+15 ms)Running coprocessor post-open hooks at 1733174082625 (+9 ms)Region opened successfully at 1733174082625 2024-12-02T21:14:42,625 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1733174081291.4197cd4f7ea2f3d6256be6836e9834da., pid=48, masterSystemTime=1733174082379 2024-12-02T21:14:42,626 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,33459,1733174078298 because future has completed 2024-12-02T21:14:42,628 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for test,zzz,1733174081291.a36d8f99c2aef77f88827585926ad989., pid=31, masterSystemTime=1733174082358 2024-12-02T21:14:42,629 INFO [StoreOpener-1a4ea0dec946dcfe96ad162d5bcf5949-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a4ea0dec946dcfe96ad162d5bcf5949 columnFamilyName family 2024-12-02T21:14:42,629 DEBUG [StoreOpener-1a4ea0dec946dcfe96ad162d5bcf5949-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:14:42,630 INFO [StoreOpener-1a4ea0dec946dcfe96ad162d5bcf5949-1 {}] regionserver.HStore(327): Store=1a4ea0dec946dcfe96ad162d5bcf5949/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T21:14:42,630 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1038): replaying wal for 1a4ea0dec946dcfe96ad162d5bcf5949 2024-12-02T21:14:42,630 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ooo,1733174081291.4197cd4f7ea2f3d6256be6836e9834da. 2024-12-02T21:14:42,630 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(153): Opened test,ooo,1733174081291.4197cd4f7ea2f3d6256be6836e9834da. 2024-12-02T21:14:42,630 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open test,uuu,1733174081291.e40e25a9323e1d3dad4ec92c578e2f82. 2024-12-02T21:14:42,631 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => e40e25a9323e1d3dad4ec92c578e2f82, NAME => 'test,uuu,1733174081291.e40e25a9323e1d3dad4ec92c578e2f82.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-12-02T21:14:42,631 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/1a4ea0dec946dcfe96ad162d5bcf5949 2024-12-02T21:14:42,631 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test e40e25a9323e1d3dad4ec92c578e2f82 2024-12-02T21:14:42,631 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=20 2024-12-02T21:14:42,631 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated test,uuu,1733174081291.e40e25a9323e1d3dad4ec92c578e2f82.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:14:42,631 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 20545c8ee2d66091a91ff9697d1f18ca, server=87c3fdb6c570,40413,1733174078061 in 378 msec 2024-12-02T21:14:42,631 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for e40e25a9323e1d3dad4ec92c578e2f82 2024-12-02T21:14:42,631 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for e40e25a9323e1d3dad4ec92c578e2f82 2024-12-02T21:14:42,632 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/1a4ea0dec946dcfe96ad162d5bcf5949 2024-12-02T21:14:42,632 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for test,zzz,1733174081291.a36d8f99c2aef77f88827585926ad989. 2024-12-02T21:14:42,632 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened test,zzz,1733174081291.a36d8f99c2aef77f88827585926ad989. 2024-12-02T21:14:42,632 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open test,kkk,1733174081291.a524d88be9ac29b416f0a6c04627fa1c. 2024-12-02T21:14:42,632 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => a524d88be9ac29b416f0a6c04627fa1c, NAME => 'test,kkk,1733174081291.a524d88be9ac29b416f0a6c04627fa1c.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-12-02T21:14:42,633 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test a524d88be9ac29b416f0a6c04627fa1c 2024-12-02T21:14:42,633 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated test,kkk,1733174081291.a524d88be9ac29b416f0a6c04627fa1c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:14:42,633 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for a524d88be9ac29b416f0a6c04627fa1c 2024-12-02T21:14:42,633 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for a524d88be9ac29b416f0a6c04627fa1c 2024-12-02T21:14:42,633 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1048): stopping wal replay for 1a4ea0dec946dcfe96ad162d5bcf5949 2024-12-02T21:14:42,633 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1060): Cleaning up temporary data for 1a4ea0dec946dcfe96ad162d5bcf5949 2024-12-02T21:14:42,633 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=a36d8f99c2aef77f88827585926ad989, regionState=OPEN, openSeqNum=2, regionLocation=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:42,635 INFO [StoreOpener-e40e25a9323e1d3dad4ec92c578e2f82-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region e40e25a9323e1d3dad4ec92c578e2f82 2024-12-02T21:14:42,635 INFO [StoreOpener-a524d88be9ac29b416f0a6c04627fa1c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region a524d88be9ac29b416f0a6c04627fa1c 2024-12-02T21:14:42,637 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): writing seq id for 1a4ea0dec946dcfe96ad162d5bcf5949 2024-12-02T21:14:42,638 INFO [StoreOpener-a524d88be9ac29b416f0a6c04627fa1c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a524d88be9ac29b416f0a6c04627fa1c columnFamilyName family 2024-12-02T21:14:42,638 DEBUG [StoreOpener-a524d88be9ac29b416f0a6c04627fa1c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:14:42,638 INFO [StoreOpener-e40e25a9323e1d3dad4ec92c578e2f82-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e40e25a9323e1d3dad4ec92c578e2f82 columnFamilyName family 2024-12-02T21:14:42,639 DEBUG [StoreOpener-e40e25a9323e1d3dad4ec92c578e2f82-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:14:42,641 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure a36d8f99c2aef77f88827585926ad989, server=87c3fdb6c570,40413,1733174078061 because future has completed 2024-12-02T21:14:42,642 INFO [StoreOpener-e40e25a9323e1d3dad4ec92c578e2f82-1 {}] regionserver.HStore(327): Store=e40e25a9323e1d3dad4ec92c578e2f82/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T21:14:42,642 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/1a4ea0dec946dcfe96ad162d5bcf5949/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T21:14:42,643 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for e40e25a9323e1d3dad4ec92c578e2f82 2024-12-02T21:14:42,643 INFO [StoreOpener-a524d88be9ac29b416f0a6c04627fa1c-1 {}] regionserver.HStore(327): Store=a524d88be9ac29b416f0a6c04627fa1c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T21:14:42,643 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1114): Opened 1a4ea0dec946dcfe96ad162d5bcf5949; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74823812, jitterRate=0.11496168375015259}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T21:14:42,644 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1a4ea0dec946dcfe96ad162d5bcf5949 2024-12-02T21:14:42,644 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1006): Region open journal for 1a4ea0dec946dcfe96ad162d5bcf5949: Running coprocessor pre-open hook at 1733174082615Writing region info on filesystem at 1733174082615Initializing all the Stores at 1733174082619 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174082620 (+1 ms)Cleaning up temporary data from old regions at 1733174082633 (+13 ms)Running coprocessor post-open hooks at 1733174082644 (+11 ms)Region opened successfully at 1733174082644 2024-12-02T21:14:42,644 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for a524d88be9ac29b416f0a6c04627fa1c 2024-12-02T21:14:42,646 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/a524d88be9ac29b416f0a6c04627fa1c 2024-12-02T21:14:42,647 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/a524d88be9ac29b416f0a6c04627fa1c 2024-12-02T21:14:42,647 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=20545c8ee2d66091a91ff9697d1f18ca, ASSIGN in 607 msec 2024-12-02T21:14:42,648 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for a524d88be9ac29b416f0a6c04627fa1c 2024-12-02T21:14:42,648 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for a524d88be9ac29b416f0a6c04627fa1c 2024-12-02T21:14:42,648 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2236): Post open deploy tasks for test,www,1733174081291.1a4ea0dec946dcfe96ad162d5bcf5949., pid=49, masterSystemTime=1733174082438 2024-12-02T21:14:42,652 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/e40e25a9323e1d3dad4ec92c578e2f82 2024-12-02T21:14:42,654 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2266): Finished post open deploy task for test,www,1733174081291.1a4ea0dec946dcfe96ad162d5bcf5949. 2024-12-02T21:14:42,654 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(153): Opened test,www,1733174081291.1a4ea0dec946dcfe96ad162d5bcf5949. 2024-12-02T21:14:42,654 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(132): Open test,nnn,1733174081291.87ba365a3e5d028c9193155af96aeb60. 2024-12-02T21:14:42,654 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7752): Opening region: {ENCODED => 87ba365a3e5d028c9193155af96aeb60, NAME => 'test,nnn,1733174081291.87ba365a3e5d028c9193155af96aeb60.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-12-02T21:14:42,655 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 87ba365a3e5d028c9193155af96aeb60 2024-12-02T21:14:42,655 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(898): Instantiated test,nnn,1733174081291.87ba365a3e5d028c9193155af96aeb60.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:14:42,655 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7794): checking encryption for 87ba365a3e5d028c9193155af96aeb60 2024-12-02T21:14:42,655 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7797): checking classloading for 87ba365a3e5d028c9193155af96aeb60 2024-12-02T21:14:42,657 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for a524d88be9ac29b416f0a6c04627fa1c 2024-12-02T21:14:42,657 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-12-02T21:14:42,657 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure a36d8f99c2aef77f88827585926ad989, server=87c3fdb6c570,40413,1733174078061 in 448 msec 2024-12-02T21:14:42,660 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/e40e25a9323e1d3dad4ec92c578e2f82 2024-12-02T21:14:42,662 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a36d8f99c2aef77f88827585926ad989, ASSIGN in 625 msec 2024-12-02T21:14:42,664 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=1a4ea0dec946dcfe96ad162d5bcf5949, regionState=OPEN, openSeqNum=2, regionLocation=87c3fdb6c570,33459,1733174078298 2024-12-02T21:14:42,665 INFO [StoreOpener-87ba365a3e5d028c9193155af96aeb60-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 87ba365a3e5d028c9193155af96aeb60 2024-12-02T21:14:42,666 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for e40e25a9323e1d3dad4ec92c578e2f82 2024-12-02T21:14:42,667 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for e40e25a9323e1d3dad4ec92c578e2f82 2024-12-02T21:14:42,668 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1a4ea0dec946dcfe96ad162d5bcf5949, server=87c3fdb6c570,33459,1733174078298 because future has completed 2024-12-02T21:14:42,669 INFO [StoreOpener-87ba365a3e5d028c9193155af96aeb60-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 87ba365a3e5d028c9193155af96aeb60 columnFamilyName family 2024-12-02T21:14:42,670 DEBUG [StoreOpener-87ba365a3e5d028c9193155af96aeb60-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:14:42,670 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=4197cd4f7ea2f3d6256be6836e9834da, regionState=OPEN, openSeqNum=2, regionLocation=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:42,671 INFO [StoreOpener-87ba365a3e5d028c9193155af96aeb60-1 {}] regionserver.HStore(327): Store=87ba365a3e5d028c9193155af96aeb60/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T21:14:42,672 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1038): replaying wal for 87ba365a3e5d028c9193155af96aeb60 2024-12-02T21:14:42,673 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/87ba365a3e5d028c9193155af96aeb60 2024-12-02T21:14:42,674 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/87ba365a3e5d028c9193155af96aeb60 2024-12-02T21:14:42,674 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for e40e25a9323e1d3dad4ec92c578e2f82 2024-12-02T21:14:42,675 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1048): stopping wal replay for 87ba365a3e5d028c9193155af96aeb60 2024-12-02T21:14:42,675 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1060): Cleaning up temporary data for 87ba365a3e5d028c9193155af96aeb60 2024-12-02T21:14:42,677 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/a524d88be9ac29b416f0a6c04627fa1c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T21:14:42,678 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=48, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4197cd4f7ea2f3d6256be6836e9834da, server=87c3fdb6c570,36585,1733174078182 because future has completed 2024-12-02T21:14:42,678 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened a524d88be9ac29b416f0a6c04627fa1c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60776584, jitterRate=-0.09435832500457764}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T21:14:42,678 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a524d88be9ac29b416f0a6c04627fa1c 2024-12-02T21:14:42,678 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for a524d88be9ac29b416f0a6c04627fa1c: Running coprocessor pre-open hook at 1733174082633Writing region info on filesystem at 1733174082633Initializing all the Stores at 1733174082634 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174082634Cleaning up temporary data from old regions at 1733174082648 (+14 ms)Running coprocessor post-open hooks at 1733174082678 (+30 ms)Region opened successfully at 1733174082678 2024-12-02T21:14:42,679 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): writing seq id for 87ba365a3e5d028c9193155af96aeb60 2024-12-02T21:14:42,680 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1733174081291.a524d88be9ac29b416f0a6c04627fa1c., pid=32, masterSystemTime=1733174082358 2024-12-02T21:14:42,683 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=27 2024-12-02T21:14:42,684 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure 1a4ea0dec946dcfe96ad162d5bcf5949, server=87c3fdb6c570,33459,1733174078298 in 364 msec 2024-12-02T21:14:42,685 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for test,kkk,1733174081291.a524d88be9ac29b416f0a6c04627fa1c. 2024-12-02T21:14:42,685 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/e40e25a9323e1d3dad4ec92c578e2f82/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T21:14:42,685 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened test,kkk,1733174081291.a524d88be9ac29b416f0a6c04627fa1c. 2024-12-02T21:14:42,686 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open test,bbb,1733174081291.5ed86f04f519cc5dccd1b6d41be3da7d. 2024-12-02T21:14:42,686 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 5ed86f04f519cc5dccd1b6d41be3da7d, NAME => 'test,bbb,1733174081291.5ed86f04f519cc5dccd1b6d41be3da7d.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-12-02T21:14:42,686 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 5ed86f04f519cc5dccd1b6d41be3da7d 2024-12-02T21:14:42,686 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened e40e25a9323e1d3dad4ec92c578e2f82; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64176658, jitterRate=-0.04369327425956726}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T21:14:42,686 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated test,bbb,1733174081291.5ed86f04f519cc5dccd1b6d41be3da7d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:14:42,687 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e40e25a9323e1d3dad4ec92c578e2f82 2024-12-02T21:14:42,687 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 5ed86f04f519cc5dccd1b6d41be3da7d 2024-12-02T21:14:42,687 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 5ed86f04f519cc5dccd1b6d41be3da7d 2024-12-02T21:14:42,687 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for e40e25a9323e1d3dad4ec92c578e2f82: Running coprocessor pre-open hook at 1733174082632Writing region info on filesystem at 1733174082632Initializing all the Stores at 1733174082633 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174082633Cleaning up temporary data from old regions at 1733174082667 (+34 ms)Running coprocessor post-open hooks at 1733174082687 (+20 ms)Region opened successfully at 1733174082687 2024-12-02T21:14:42,688 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1733174081291.e40e25a9323e1d3dad4ec92c578e2f82., pid=34, masterSystemTime=1733174082379 2024-12-02T21:14:42,688 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN in 651 msec 2024-12-02T21:14:42,691 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=a524d88be9ac29b416f0a6c04627fa1c, regionState=OPEN, openSeqNum=2, regionLocation=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:42,693 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/87ba365a3e5d028c9193155af96aeb60/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T21:14:42,693 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for test,uuu,1733174081291.e40e25a9323e1d3dad4ec92c578e2f82. 2024-12-02T21:14:42,693 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened test,uuu,1733174081291.e40e25a9323e1d3dad4ec92c578e2f82. 2024-12-02T21:14:42,693 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42045 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=87c3fdb6c570,40413,1733174078061, table=test, region=a524d88be9ac29b416f0a6c04627fa1c. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-02T21:14:42,693 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open test,yyy,1733174081291.0875ddf9cbd035a3345be0684b07b2e6. 2024-12-02T21:14:42,694 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => 0875ddf9cbd035a3345be0684b07b2e6, NAME => 'test,yyy,1733174081291.0875ddf9cbd035a3345be0684b07b2e6.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-12-02T21:14:42,694 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1114): Opened 87ba365a3e5d028c9193155af96aeb60; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68360780, jitterRate=0.018655002117156982}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T21:14:42,694 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 0875ddf9cbd035a3345be0684b07b2e6 2024-12-02T21:14:42,694 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42045 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=87c3fdb6c570,36585,1733174078182, table=test, region=4197cd4f7ea2f3d6256be6836e9834da. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-02T21:14:42,694 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 87ba365a3e5d028c9193155af96aeb60 2024-12-02T21:14:42,694 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated test,yyy,1733174081291.0875ddf9cbd035a3345be0684b07b2e6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:14:42,694 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1006): Region open journal for 87ba365a3e5d028c9193155af96aeb60: Running coprocessor pre-open hook at 1733174082656Writing region info on filesystem at 1733174082656Initializing all the Stores at 1733174082658 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174082658Cleaning up temporary data from old regions at 1733174082675 (+17 ms)Running coprocessor post-open hooks at 1733174082694 (+19 ms)Region opened successfully at 1733174082694 2024-12-02T21:14:42,694 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for 0875ddf9cbd035a3345be0684b07b2e6 2024-12-02T21:14:42,694 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for 0875ddf9cbd035a3345be0684b07b2e6 2024-12-02T21:14:42,695 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2236): Post open deploy tasks for test,nnn,1733174081291.87ba365a3e5d028c9193155af96aeb60., pid=52, masterSystemTime=1733174082438 2024-12-02T21:14:42,699 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure a524d88be9ac29b416f0a6c04627fa1c, server=87c3fdb6c570,40413,1733174078061 because future has completed 2024-12-02T21:14:42,701 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2266): Finished post open deploy task for test,nnn,1733174081291.87ba365a3e5d028c9193155af96aeb60. 2024-12-02T21:14:42,701 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(153): Opened test,nnn,1733174081291.87ba365a3e5d028c9193155af96aeb60. 2024-12-02T21:14:42,701 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open test,qqq,1733174081291.a50bae1d5cb3d03a86dc103b09a0f68b. 2024-12-02T21:14:42,701 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => a50bae1d5cb3d03a86dc103b09a0f68b, NAME => 'test,qqq,1733174081291.a50bae1d5cb3d03a86dc103b09a0f68b.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-12-02T21:14:42,702 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test a50bae1d5cb3d03a86dc103b09a0f68b 2024-12-02T21:14:42,702 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated test,qqq,1733174081291.a50bae1d5cb3d03a86dc103b09a0f68b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:14:42,702 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for a50bae1d5cb3d03a86dc103b09a0f68b 2024-12-02T21:14:42,702 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for a50bae1d5cb3d03a86dc103b09a0f68b 2024-12-02T21:14:42,705 INFO [StoreOpener-5ed86f04f519cc5dccd1b6d41be3da7d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 5ed86f04f519cc5dccd1b6d41be3da7d 2024-12-02T21:14:42,707 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42045 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=87c3fdb6c570,33459,1733174078298, table=test, region=0d0be65f6ffd7ebb4c8b2fee37042136. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-02T21:14:42,707 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42045 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=87c3fdb6c570,33459,1733174078298, table=test, region=d63da23e8236e18b416716ec838dd43e. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-02T21:14:42,708 INFO [StoreOpener-5ed86f04f519cc5dccd1b6d41be3da7d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5ed86f04f519cc5dccd1b6d41be3da7d columnFamilyName family 2024-12-02T21:14:42,708 DEBUG [StoreOpener-5ed86f04f519cc5dccd1b6d41be3da7d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:14:42,709 INFO [StoreOpener-5ed86f04f519cc5dccd1b6d41be3da7d-1 {}] regionserver.HStore(327): Store=5ed86f04f519cc5dccd1b6d41be3da7d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T21:14:42,709 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 5ed86f04f519cc5dccd1b6d41be3da7d 2024-12-02T21:14:42,710 INFO [StoreOpener-0875ddf9cbd035a3345be0684b07b2e6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 0875ddf9cbd035a3345be0684b07b2e6 2024-12-02T21:14:42,711 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/5ed86f04f519cc5dccd1b6d41be3da7d 2024-12-02T21:14:42,712 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/5ed86f04f519cc5dccd1b6d41be3da7d 2024-12-02T21:14:42,713 INFO [StoreOpener-0875ddf9cbd035a3345be0684b07b2e6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0875ddf9cbd035a3345be0684b07b2e6 columnFamilyName family 2024-12-02T21:14:42,713 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 5ed86f04f519cc5dccd1b6d41be3da7d 2024-12-02T21:14:42,714 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 5ed86f04f519cc5dccd1b6d41be3da7d 2024-12-02T21:14:42,714 DEBUG [StoreOpener-0875ddf9cbd035a3345be0684b07b2e6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:14:42,714 INFO [StoreOpener-0875ddf9cbd035a3345be0684b07b2e6-1 {}] regionserver.HStore(327): Store=0875ddf9cbd035a3345be0684b07b2e6/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T21:14:42,714 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for 0875ddf9cbd035a3345be0684b07b2e6 2024-12-02T21:14:42,716 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/0875ddf9cbd035a3345be0684b07b2e6 2024-12-02T21:14:42,716 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/0875ddf9cbd035a3345be0684b07b2e6 2024-12-02T21:14:42,717 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for 0875ddf9cbd035a3345be0684b07b2e6 2024-12-02T21:14:42,717 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 5ed86f04f519cc5dccd1b6d41be3da7d 2024-12-02T21:14:42,717 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for 0875ddf9cbd035a3345be0684b07b2e6 2024-12-02T21:14:42,720 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=15 2024-12-02T21:14:42,721 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure a524d88be9ac29b416f0a6c04627fa1c, server=87c3fdb6c570,40413,1733174078061 in 495 msec 2024-12-02T21:14:42,721 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for 0875ddf9cbd035a3345be0684b07b2e6 2024-12-02T21:14:42,725 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a524d88be9ac29b416f0a6c04627fa1c, ASSIGN in 688 msec 2024-12-02T21:14:42,728 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=87ba365a3e5d028c9193155af96aeb60, regionState=OPEN, openSeqNum=2, regionLocation=87c3fdb6c570,33459,1733174078298 2024-12-02T21:14:42,731 INFO [StoreOpener-a50bae1d5cb3d03a86dc103b09a0f68b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region a50bae1d5cb3d03a86dc103b09a0f68b 2024-12-02T21:14:42,732 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 87ba365a3e5d028c9193155af96aeb60, server=87c3fdb6c570,33459,1733174078298 because future has completed 2024-12-02T21:14:42,733 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/5ed86f04f519cc5dccd1b6d41be3da7d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T21:14:42,734 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 5ed86f04f519cc5dccd1b6d41be3da7d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63349238, jitterRate=-0.05602279305458069}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T21:14:42,734 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5ed86f04f519cc5dccd1b6d41be3da7d 2024-12-02T21:14:42,734 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 5ed86f04f519cc5dccd1b6d41be3da7d: Running coprocessor pre-open hook at 1733174082687Writing region info on filesystem at 1733174082687Initializing all the Stores at 1733174082693 (+6 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174082693Cleaning up temporary data from old regions at 1733174082714 (+21 ms)Running coprocessor post-open hooks at 1733174082734 (+20 ms)Region opened successfully at 1733174082734 2024-12-02T21:14:42,736 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for test,bbb,1733174081291.5ed86f04f519cc5dccd1b6d41be3da7d., pid=33, masterSystemTime=1733174082358 2024-12-02T21:14:42,736 INFO [StoreOpener-a50bae1d5cb3d03a86dc103b09a0f68b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a50bae1d5cb3d03a86dc103b09a0f68b columnFamilyName family 2024-12-02T21:14:42,736 DEBUG [StoreOpener-a50bae1d5cb3d03a86dc103b09a0f68b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:14:42,737 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/0875ddf9cbd035a3345be0684b07b2e6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T21:14:42,738 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened 0875ddf9cbd035a3345be0684b07b2e6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59121653, jitterRate=-0.11901871860027313}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T21:14:42,739 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0875ddf9cbd035a3345be0684b07b2e6 2024-12-02T21:14:42,739 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for 0875ddf9cbd035a3345be0684b07b2e6: Running coprocessor pre-open hook at 1733174082694Writing region info on filesystem at 1733174082694Initializing all the Stores at 1733174082696 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174082696Cleaning up temporary data from old regions at 1733174082717 (+21 ms)Running coprocessor post-open hooks at 1733174082739 (+22 ms)Region opened successfully at 1733174082739 2024-12-02T21:14:42,739 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=e40e25a9323e1d3dad4ec92c578e2f82, regionState=OPEN, openSeqNum=2, regionLocation=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:42,740 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for test,bbb,1733174081291.5ed86f04f519cc5dccd1b6d41be3da7d. 2024-12-02T21:14:42,740 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1733174081291.0875ddf9cbd035a3345be0684b07b2e6., pid=35, masterSystemTime=1733174082379 2024-12-02T21:14:42,740 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened test,bbb,1733174081291.5ed86f04f519cc5dccd1b6d41be3da7d. 2024-12-02T21:14:42,743 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure e40e25a9323e1d3dad4ec92c578e2f82, server=87c3fdb6c570,36585,1733174078182 because future has completed 2024-12-02T21:14:42,745 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1733174081291.0875ddf9cbd035a3345be0684b07b2e6. 2024-12-02T21:14:42,745 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened test,yyy,1733174081291.0875ddf9cbd035a3345be0684b07b2e6. 2024-12-02T21:14:42,743 INFO [StoreOpener-a50bae1d5cb3d03a86dc103b09a0f68b-1 {}] regionserver.HStore(327): Store=a50bae1d5cb3d03a86dc103b09a0f68b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T21:14:42,745 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open test,iii,1733174081291.8e039115fcc0d0a54e2d03efed7e05de. 2024-12-02T21:14:42,745 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for a50bae1d5cb3d03a86dc103b09a0f68b 2024-12-02T21:14:42,745 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => 8e039115fcc0d0a54e2d03efed7e05de, NAME => 'test,iii,1733174081291.8e039115fcc0d0a54e2d03efed7e05de.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-12-02T21:14:42,746 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8e039115fcc0d0a54e2d03efed7e05de 2024-12-02T21:14:42,746 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated test,iii,1733174081291.8e039115fcc0d0a54e2d03efed7e05de.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:14:42,746 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for 8e039115fcc0d0a54e2d03efed7e05de 2024-12-02T21:14:42,746 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for 8e039115fcc0d0a54e2d03efed7e05de 2024-12-02T21:14:42,746 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/a50bae1d5cb3d03a86dc103b09a0f68b 2024-12-02T21:14:42,747 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/a50bae1d5cb3d03a86dc103b09a0f68b 2024-12-02T21:14:42,748 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for a50bae1d5cb3d03a86dc103b09a0f68b 2024-12-02T21:14:42,749 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for a50bae1d5cb3d03a86dc103b09a0f68b 2024-12-02T21:14:42,752 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for a50bae1d5cb3d03a86dc103b09a0f68b 2024-12-02T21:14:42,752 INFO [StoreOpener-8e039115fcc0d0a54e2d03efed7e05de-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8e039115fcc0d0a54e2d03efed7e05de 2024-12-02T21:14:42,757 INFO [StoreOpener-8e039115fcc0d0a54e2d03efed7e05de-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8e039115fcc0d0a54e2d03efed7e05de columnFamilyName family 2024-12-02T21:14:42,757 DEBUG [StoreOpener-8e039115fcc0d0a54e2d03efed7e05de-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:14:42,758 INFO [StoreOpener-8e039115fcc0d0a54e2d03efed7e05de-1 {}] regionserver.HStore(327): Store=8e039115fcc0d0a54e2d03efed7e05de/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T21:14:42,759 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for 8e039115fcc0d0a54e2d03efed7e05de 2024-12-02T21:14:42,760 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/8e039115fcc0d0a54e2d03efed7e05de 2024-12-02T21:14:42,761 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/8e039115fcc0d0a54e2d03efed7e05de 2024-12-02T21:14:42,762 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/a50bae1d5cb3d03a86dc103b09a0f68b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T21:14:42,762 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for 8e039115fcc0d0a54e2d03efed7e05de 2024-12-02T21:14:42,762 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for 8e039115fcc0d0a54e2d03efed7e05de 2024-12-02T21:14:42,763 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened a50bae1d5cb3d03a86dc103b09a0f68b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65329220, jitterRate=-0.02651876211166382}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T21:14:42,763 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a50bae1d5cb3d03a86dc103b09a0f68b 2024-12-02T21:14:42,763 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for a50bae1d5cb3d03a86dc103b09a0f68b: Running coprocessor pre-open hook at 1733174082702Writing region info on filesystem at 1733174082702Initializing all the Stores at 1733174082704 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174082704Cleaning up temporary data from old regions at 1733174082749 (+45 ms)Running coprocessor post-open hooks at 1733174082763 (+14 ms)Region opened successfully at 1733174082763 2024-12-02T21:14:42,765 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for test,qqq,1733174081291.a50bae1d5cb3d03a86dc103b09a0f68b., pid=53, masterSystemTime=1733174082438 2024-12-02T21:14:42,766 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=25 2024-12-02T21:14:42,766 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure e40e25a9323e1d3dad4ec92c578e2f82, server=87c3fdb6c570,36585,1733174078182 in 531 msec 2024-12-02T21:14:42,766 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for 8e039115fcc0d0a54e2d03efed7e05de 2024-12-02T21:14:42,770 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/8e039115fcc0d0a54e2d03efed7e05de/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T21:14:42,771 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for test,qqq,1733174081291.a50bae1d5cb3d03a86dc103b09a0f68b. 2024-12-02T21:14:42,771 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened test,qqq,1733174081291.a50bae1d5cb3d03a86dc103b09a0f68b. 2024-12-02T21:14:42,771 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open test,ggg,1733174081291.0dfd4169e4c254c51bcc70a472f99ecf. 2024-12-02T21:14:42,771 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened 8e039115fcc0d0a54e2d03efed7e05de; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63087482, jitterRate=-0.059923261404037476}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T21:14:42,771 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8e039115fcc0d0a54e2d03efed7e05de 2024-12-02T21:14:42,771 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => 0dfd4169e4c254c51bcc70a472f99ecf, NAME => 'test,ggg,1733174081291.0dfd4169e4c254c51bcc70a472f99ecf.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-12-02T21:14:42,771 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for 8e039115fcc0d0a54e2d03efed7e05de: Running coprocessor pre-open hook at 1733174082746Writing region info on filesystem at 1733174082746Initializing all the Stores at 1733174082750 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174082750Cleaning up temporary data from old regions at 1733174082763 (+13 ms)Running coprocessor post-open hooks at 1733174082771 (+8 ms)Region opened successfully at 1733174082771 2024-12-02T21:14:42,772 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 0dfd4169e4c254c51bcc70a472f99ecf 2024-12-02T21:14:42,772 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated test,ggg,1733174081291.0dfd4169e4c254c51bcc70a472f99ecf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:14:42,772 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for 0dfd4169e4c254c51bcc70a472f99ecf 2024-12-02T21:14:42,772 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for 0dfd4169e4c254c51bcc70a472f99ecf 2024-12-02T21:14:42,773 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1733174081291.8e039115fcc0d0a54e2d03efed7e05de., pid=47, masterSystemTime=1733174082379 2024-12-02T21:14:42,774 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e40e25a9323e1d3dad4ec92c578e2f82, ASSIGN in 733 msec 2024-12-02T21:14:42,776 INFO [StoreOpener-0dfd4169e4c254c51bcc70a472f99ecf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 0dfd4169e4c254c51bcc70a472f99ecf 2024-12-02T21:14:42,776 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=a50bae1d5cb3d03a86dc103b09a0f68b, regionState=OPEN, openSeqNum=2, regionLocation=87c3fdb6c570,33459,1733174078298 2024-12-02T21:14:42,779 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1733174081291.8e039115fcc0d0a54e2d03efed7e05de. 2024-12-02T21:14:42,779 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened test,iii,1733174081291.8e039115fcc0d0a54e2d03efed7e05de. 2024-12-02T21:14:42,780 INFO [StoreOpener-0dfd4169e4c254c51bcc70a472f99ecf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0dfd4169e4c254c51bcc70a472f99ecf columnFamilyName family 2024-12-02T21:14:42,780 DEBUG [StoreOpener-0dfd4169e4c254c51bcc70a472f99ecf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:14:42,782 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure a50bae1d5cb3d03a86dc103b09a0f68b, server=87c3fdb6c570,33459,1733174078298 because future has completed 2024-12-02T21:14:42,782 INFO [StoreOpener-0dfd4169e4c254c51bcc70a472f99ecf-1 {}] regionserver.HStore(327): Store=0dfd4169e4c254c51bcc70a472f99ecf/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T21:14:42,782 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=8e039115fcc0d0a54e2d03efed7e05de, regionState=OPEN, openSeqNum=2, regionLocation=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:42,782 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for 0dfd4169e4c254c51bcc70a472f99ecf 2024-12-02T21:14:42,784 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/0dfd4169e4c254c51bcc70a472f99ecf 2024-12-02T21:14:42,785 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/0dfd4169e4c254c51bcc70a472f99ecf 2024-12-02T21:14:42,786 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for 0dfd4169e4c254c51bcc70a472f99ecf 2024-12-02T21:14:42,786 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for 0dfd4169e4c254c51bcc70a472f99ecf 2024-12-02T21:14:42,787 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8e039115fcc0d0a54e2d03efed7e05de, server=87c3fdb6c570,36585,1733174078182 because future has completed 2024-12-02T21:14:42,790 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for 0dfd4169e4c254c51bcc70a472f99ecf 2024-12-02T21:14:42,792 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=21 2024-12-02T21:14:42,793 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=21, state=SUCCESS, hasLock=false; OpenRegionProcedure a50bae1d5cb3d03a86dc103b09a0f68b, server=87c3fdb6c570,33459,1733174078298 in 459 msec 2024-12-02T21:14:42,794 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/0dfd4169e4c254c51bcc70a472f99ecf/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T21:14:42,795 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened 0dfd4169e4c254c51bcc70a472f99ecf; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66529990, jitterRate=-0.008625894784927368}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T21:14:42,796 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0dfd4169e4c254c51bcc70a472f99ecf 2024-12-02T21:14:42,796 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for 0dfd4169e4c254c51bcc70a472f99ecf: Running coprocessor pre-open hook at 1733174082773Writing region info on filesystem at 1733174082773Initializing all the Stores at 1733174082775 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174082775Cleaning up temporary data from old regions at 1733174082786 (+11 ms)Running coprocessor post-open hooks at 1733174082796 (+10 ms)Region opened successfully at 1733174082796 2024-12-02T21:14:42,796 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b, ASSIGN in 760 msec 2024-12-02T21:14:42,798 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ggg,1733174081291.0dfd4169e4c254c51bcc70a472f99ecf., pid=51, masterSystemTime=1733174082438 2024-12-02T21:14:42,798 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42045 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=87c3fdb6c570,36585,1733174078182, table=test, region=8e039115fcc0d0a54e2d03efed7e05de. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-02T21:14:42,798 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42045 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=87c3fdb6c570,36585,1733174078182, table=test, region=4197cd4f7ea2f3d6256be6836e9834da. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-02T21:14:42,803 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ggg,1733174081291.0dfd4169e4c254c51bcc70a472f99ecf. 2024-12-02T21:14:42,803 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened test,ggg,1733174081291.0dfd4169e4c254c51bcc70a472f99ecf. 2024-12-02T21:14:42,803 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(132): Open test,ccc,1733174081291.0517912ff18e81317bb5474cefaca025. 2024-12-02T21:14:42,803 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7752): Opening region: {ENCODED => 0517912ff18e81317bb5474cefaca025, NAME => 'test,ccc,1733174081291.0517912ff18e81317bb5474cefaca025.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-12-02T21:14:42,804 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 0517912ff18e81317bb5474cefaca025 2024-12-02T21:14:42,804 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(898): Instantiated test,ccc,1733174081291.0517912ff18e81317bb5474cefaca025.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:14:42,805 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7794): checking encryption for 0517912ff18e81317bb5474cefaca025 2024-12-02T21:14:42,805 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7797): checking classloading for 0517912ff18e81317bb5474cefaca025 2024-12-02T21:14:42,805 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=13 2024-12-02T21:14:42,806 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure 8e039115fcc0d0a54e2d03efed7e05de, server=87c3fdb6c570,36585,1733174078182 in 502 msec 2024-12-02T21:14:42,810 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8e039115fcc0d0a54e2d03efed7e05de, ASSIGN in 773 msec 2024-12-02T21:14:42,811 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42045 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=87c3fdb6c570,33459,1733174078298, table=test, region=0d0be65f6ffd7ebb4c8b2fee37042136. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-02T21:14:42,811 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42045 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=87c3fdb6c570,33459,1733174078298, table=test, region=87ba365a3e5d028c9193155af96aeb60. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-02T21:14:42,812 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42045 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=87c3fdb6c570,33459,1733174078298, table=test, region=d63da23e8236e18b416716ec838dd43e. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-02T21:14:42,812 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=0dfd4169e4c254c51bcc70a472f99ecf, regionState=OPEN, openSeqNum=2, regionLocation=87c3fdb6c570,33459,1733174078298 2024-12-02T21:14:42,816 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0dfd4169e4c254c51bcc70a472f99ecf, server=87c3fdb6c570,33459,1733174078298 because future has completed 2024-12-02T21:14:42,817 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=0875ddf9cbd035a3345be0684b07b2e6, regionState=OPEN, openSeqNum=2, regionLocation=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:42,823 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0875ddf9cbd035a3345be0684b07b2e6, server=87c3fdb6c570,36585,1733174078182 because future has completed 2024-12-02T21:14:42,829 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=29 2024-12-02T21:14:42,829 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 0875ddf9cbd035a3345be0684b07b2e6, server=87c3fdb6c570,36585,1733174078182 in 595 msec 2024-12-02T21:14:42,832 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=0875ddf9cbd035a3345be0684b07b2e6, ASSIGN in 797 msec 2024-12-02T21:14:42,837 INFO [StoreOpener-0517912ff18e81317bb5474cefaca025-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 0517912ff18e81317bb5474cefaca025 2024-12-02T21:14:42,839 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=11 2024-12-02T21:14:42,839 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 0dfd4169e4c254c51bcc70a472f99ecf, server=87c3fdb6c570,33459,1733174078298 in 513 msec 2024-12-02T21:14:42,842 INFO [StoreOpener-0517912ff18e81317bb5474cefaca025-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0517912ff18e81317bb5474cefaca025 columnFamilyName family 2024-12-02T21:14:42,842 DEBUG [StoreOpener-0517912ff18e81317bb5474cefaca025-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:14:42,843 INFO [StoreOpener-0517912ff18e81317bb5474cefaca025-1 {}] regionserver.HStore(327): Store=0517912ff18e81317bb5474cefaca025/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T21:14:42,843 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1038): replaying wal for 0517912ff18e81317bb5474cefaca025 2024-12-02T21:14:42,845 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/0517912ff18e81317bb5474cefaca025 2024-12-02T21:14:42,845 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=0dfd4169e4c254c51bcc70a472f99ecf, ASSIGN in 807 msec 2024-12-02T21:14:42,845 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/0517912ff18e81317bb5474cefaca025 2024-12-02T21:14:42,847 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1048): stopping wal replay for 0517912ff18e81317bb5474cefaca025 2024-12-02T21:14:42,847 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1060): Cleaning up temporary data for 0517912ff18e81317bb5474cefaca025 2024-12-02T21:14:42,849 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=5ed86f04f519cc5dccd1b6d41be3da7d, regionState=OPEN, openSeqNum=2, regionLocation=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:42,854 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1093): writing seq id for 0517912ff18e81317bb5474cefaca025 2024-12-02T21:14:42,856 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5ed86f04f519cc5dccd1b6d41be3da7d, server=87c3fdb6c570,40413,1733174078061 because future has completed 2024-12-02T21:14:42,857 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/0517912ff18e81317bb5474cefaca025/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T21:14:42,858 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1114): Opened 0517912ff18e81317bb5474cefaca025; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64848603, jitterRate=-0.0336805135011673}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T21:14:42,859 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0517912ff18e81317bb5474cefaca025 2024-12-02T21:14:42,859 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1006): Region open journal for 0517912ff18e81317bb5474cefaca025: Running coprocessor pre-open hook at 1733174082805Writing region info on filesystem at 1733174082805Initializing all the Stores at 1733174082821 (+16 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174082821Cleaning up temporary data from old regions at 1733174082847 (+26 ms)Running coprocessor post-open hooks at 1733174082859 (+12 ms)Region opened successfully at 1733174082859 2024-12-02T21:14:42,860 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1733174081291.0517912ff18e81317bb5474cefaca025., pid=56, masterSystemTime=1733174082438 2024-12-02T21:14:42,860 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=18 2024-12-02T21:14:42,860 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure 87ba365a3e5d028c9193155af96aeb60, server=87c3fdb6c570,33459,1733174078298 in 531 msec 2024-12-02T21:14:42,862 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=87ba365a3e5d028c9193155af96aeb60, ASSIGN in 827 msec 2024-12-02T21:14:42,863 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1733174081291.0517912ff18e81317bb5474cefaca025. 2024-12-02T21:14:42,863 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(153): Opened test,ccc,1733174081291.0517912ff18e81317bb5474cefaca025. 2024-12-02T21:14:42,863 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open test,eee,1733174081291.120cf3776e92a15d3276f11911c4042f. 2024-12-02T21:14:42,864 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => 120cf3776e92a15d3276f11911c4042f, NAME => 'test,eee,1733174081291.120cf3776e92a15d3276f11911c4042f.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-12-02T21:14:42,864 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 120cf3776e92a15d3276f11911c4042f 2024-12-02T21:14:42,864 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated test,eee,1733174081291.120cf3776e92a15d3276f11911c4042f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:14:42,864 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=0517912ff18e81317bb5474cefaca025, regionState=OPEN, openSeqNum=2, regionLocation=87c3fdb6c570,33459,1733174078298 2024-12-02T21:14:42,864 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for 120cf3776e92a15d3276f11911c4042f 2024-12-02T21:14:42,864 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for 120cf3776e92a15d3276f11911c4042f 2024-12-02T21:14:42,868 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0517912ff18e81317bb5474cefaca025, server=87c3fdb6c570,33459,1733174078298 because future has completed 2024-12-02T21:14:42,868 INFO [StoreOpener-120cf3776e92a15d3276f11911c4042f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 120cf3776e92a15d3276f11911c4042f 2024-12-02T21:14:42,871 INFO [StoreOpener-120cf3776e92a15d3276f11911c4042f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 120cf3776e92a15d3276f11911c4042f columnFamilyName family 2024-12-02T21:14:42,871 DEBUG [StoreOpener-120cf3776e92a15d3276f11911c4042f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:14:42,872 INFO [StoreOpener-120cf3776e92a15d3276f11911c4042f-1 {}] regionserver.HStore(327): Store=120cf3776e92a15d3276f11911c4042f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T21:14:42,872 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=6 2024-12-02T21:14:42,872 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for 120cf3776e92a15d3276f11911c4042f 2024-12-02T21:14:42,872 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure 5ed86f04f519cc5dccd1b6d41be3da7d, server=87c3fdb6c570,40413,1733174078061 in 649 msec 2024-12-02T21:14:42,873 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/120cf3776e92a15d3276f11911c4042f 2024-12-02T21:14:42,874 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/120cf3776e92a15d3276f11911c4042f 2024-12-02T21:14:42,875 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for 120cf3776e92a15d3276f11911c4042f 2024-12-02T21:14:42,875 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for 120cf3776e92a15d3276f11911c4042f 2024-12-02T21:14:42,875 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=5ed86f04f519cc5dccd1b6d41be3da7d, ASSIGN in 839 msec 2024-12-02T21:14:42,877 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for 120cf3776e92a15d3276f11911c4042f 2024-12-02T21:14:42,880 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=7 2024-12-02T21:14:42,880 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 0517912ff18e81317bb5474cefaca025, server=87c3fdb6c570,33459,1733174078298 in 534 msec 2024-12-02T21:14:42,881 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/120cf3776e92a15d3276f11911c4042f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T21:14:42,881 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened 120cf3776e92a15d3276f11911c4042f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60147012, jitterRate=-0.1037396788597107}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T21:14:42,882 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 120cf3776e92a15d3276f11911c4042f 2024-12-02T21:14:42,882 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for 120cf3776e92a15d3276f11911c4042f: Running coprocessor pre-open hook at 1733174082864Writing region info on filesystem at 1733174082864Initializing all the Stores at 1733174082866 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174082866Cleaning up temporary data from old regions at 1733174082875 (+9 ms)Running coprocessor post-open hooks at 1733174082882 (+7 ms)Region opened successfully at 1733174082882 2024-12-02T21:14:42,883 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1733174081291.120cf3776e92a15d3276f11911c4042f., pid=55, masterSystemTime=1733174082438 2024-12-02T21:14:42,886 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN in 848 msec 2024-12-02T21:14:42,887 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1733174081291.120cf3776e92a15d3276f11911c4042f. 2024-12-02T21:14:42,887 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened test,eee,1733174081291.120cf3776e92a15d3276f11911c4042f. 2024-12-02T21:14:42,888 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=120cf3776e92a15d3276f11911c4042f, regionState=OPEN, openSeqNum=2, regionLocation=87c3fdb6c570,33459,1733174078298 2024-12-02T21:14:42,891 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 120cf3776e92a15d3276f11911c4042f, server=87c3fdb6c570,33459,1733174078298 because future has completed 2024-12-02T21:14:42,894 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=19 2024-12-02T21:14:42,894 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=19, state=SUCCESS, hasLock=false; OpenRegionProcedure 4197cd4f7ea2f3d6256be6836e9834da, server=87c3fdb6c570,36585,1733174078182 in 587 msec 2024-12-02T21:14:42,896 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=4197cd4f7ea2f3d6256be6836e9834da, ASSIGN in 861 msec 2024-12-02T21:14:42,902 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=9 2024-12-02T21:14:42,902 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure 120cf3776e92a15d3276f11911c4042f, server=87c3fdb6c570,33459,1733174078298 in 560 msec 2024-12-02T21:14:42,905 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=120cf3776e92a15d3276f11911c4042f, ASSIGN in 870 msec 2024-12-02T21:14:42,909 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=24 2024-12-02T21:14:42,909 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,33459,1733174078298 in 574 msec 2024-12-02T21:14:42,912 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d63da23e8236e18b416716ec838dd43e, ASSIGN in 877 msec 2024-12-02T21:14:42,918 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=16 2024-12-02T21:14:42,918 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 0d0be65f6ffd7ebb4c8b2fee37042136, server=87c3fdb6c570,33459,1733174078298 in 598 msec 2024-12-02T21:14:42,922 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=4 2024-12-02T21:14:42,922 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=0d0be65f6ffd7ebb4c8b2fee37042136, ASSIGN in 885 msec 2024-12-02T21:14:42,924 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-02T21:14:42,924 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733174082924"}]},"ts":"1733174082924"} 2024-12-02T21:14:42,927 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLED in hbase:meta 2024-12-02T21:14:42,929 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_POST_OPERATION 2024-12-02T21:14:42,934 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test in 1.6250 sec 2024-12-02T21:14:43,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42045 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T21:14:43,473 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table test get assigned. Timeout = 60000ms 2024-12-02T21:14:43,473 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test completed 2024-12-02T21:14:43,475 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-02T21:14:43,496 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table test assigned to meta. Checking AM states. 2024-12-02T21:14:43,496 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-02T21:14:43,497 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table test assigned. 2024-12-02T21:14:43,498 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-12-02T21:14:43,504 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-12-02T21:14:43,528 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-12-02T21:14:43,544 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:14:43,546 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40262, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:14:43,548 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:14:43,550 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56836, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:14:43,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42045 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 0517912ff18e81317bb5474cefaca025 changed from -1.0 to 0.0, refreshing cache 2024-12-02T21:14:43,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42045 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 120cf3776e92a15d3276f11911c4042f changed from -1.0 to 0.0, refreshing cache 2024-12-02T21:14:43,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42045 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 0dfd4169e4c254c51bcc70a472f99ecf changed from -1.0 to 0.0, refreshing cache 2024-12-02T21:14:43,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42045 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 0d0be65f6ffd7ebb4c8b2fee37042136 changed from -1.0 to 0.0, refreshing cache 2024-12-02T21:14:43,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42045 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 87ba365a3e5d028c9193155af96aeb60 changed from -1.0 to 0.0, refreshing cache 2024-12-02T21:14:43,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42045 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region a50bae1d5cb3d03a86dc103b09a0f68b changed from -1.0 to 0.0, refreshing cache 2024-12-02T21:14:43,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42045 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region d63da23e8236e18b416716ec838dd43e changed from -1.0 to 0.0, refreshing cache 2024-12-02T21:14:43,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42045 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1a4ea0dec946dcfe96ad162d5bcf5949 changed from -1.0 to 0.0, refreshing cache 2024-12-02T21:14:43,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42045 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 06d131e39cb4f5b02db4ec472d438a26 changed from -1.0 to 0.0, refreshing cache 2024-12-02T21:14:43,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42045 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 85ca3a9a09d75c0941c58f5b83584a8e changed from -1.0 to 0.0, refreshing cache 2024-12-02T21:14:43,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42045 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region e87d99b937228303e28b7ea0373ddc70 changed from -1.0 to 0.0, refreshing cache 2024-12-02T21:14:43,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42045 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 8e039115fcc0d0a54e2d03efed7e05de changed from -1.0 to 0.0, refreshing cache 2024-12-02T21:14:43,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42045 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 5d60c566b39516dd875aaf2bd7def402 changed from -1.0 to 0.0, refreshing cache 2024-12-02T21:14:43,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42045 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 4197cd4f7ea2f3d6256be6836e9834da changed from -1.0 to 0.0, refreshing cache 2024-12-02T21:14:43,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42045 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 215753e705337ae11b1b38ad77b6a5cd changed from -1.0 to 0.0, refreshing cache 2024-12-02T21:14:43,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42045 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region e40e25a9323e1d3dad4ec92c578e2f82 changed from -1.0 to 0.0, refreshing cache 2024-12-02T21:14:43,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42045 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 0875ddf9cbd035a3345be0684b07b2e6 changed from -1.0 to 0.0, refreshing cache 2024-12-02T21:14:43,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42045 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 5ed86f04f519cc5dccd1b6d41be3da7d changed from -1.0 to 0.0, refreshing cache 2024-12-02T21:14:43,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42045 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ff611163b4ab91c50d940651a0065c34 changed from -1.0 to 0.0, refreshing cache 2024-12-02T21:14:43,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42045 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 9950441eda7548fa66872811a13f5bf0 changed from -1.0 to 0.0, refreshing cache 2024-12-02T21:14:43,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42045 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region a524d88be9ac29b416f0a6c04627fa1c changed from -1.0 to 0.0, refreshing cache 2024-12-02T21:14:43,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42045 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 292fd0b4e0f07516cc25c069181e3111 changed from -1.0 to 0.0, refreshing cache 2024-12-02T21:14:43,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42045 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 20545c8ee2d66091a91ff9697d1f18ca changed from -1.0 to 0.0, refreshing cache 2024-12-02T21:14:43,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42045 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region e42e4828e881f12c74fbedcdb6b00ea2 changed from -1.0 to 0.0, refreshing cache 2024-12-02T21:14:43,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42045 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 6a2fb6e9d39bf04774472d3b4c70810a changed from -1.0 to 0.0, refreshing cache 2024-12-02T21:14:43,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42045 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region a36d8f99c2aef77f88827585926ad989 changed from -1.0 to 0.0, refreshing cache 2024-12-02T21:14:43,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42045 {}] rsgroup.RSGroupBasedLoadBalancer(138): Balancing RSGroup=default 2024-12-02T21:14:43,611 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42045 {}] rsgroup.RSGroupBasedLoadBalancer(151): Start Generate Balance plan for group: default 2024-12-02T21:14:43,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42045 {}] balancer.BaseLoadBalancer(619): Start Generate Balance plan for cluster. 2024-12-02T21:14:43,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42045 {}] balancer.BalancerClusterState(204): Hosts are {87c3fdb6c570=0} racks are {/default-rack=0} 2024-12-02T21:14:43,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42045 {}] balancer.BalancerClusterState(303): server 0 has 10 regions 2024-12-02T21:14:43,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42045 {}] balancer.BalancerClusterState(303): server 1 has 8 regions 2024-12-02T21:14:43,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42045 {}] balancer.BalancerClusterState(303): server 2 has 9 regions 2024-12-02T21:14:43,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42045 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-02T21:14:43,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42045 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-02T21:14:43,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42045 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-02T21:14:43,613 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42045 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-02T21:14:43,613 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42045 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-02T21:14:43,613 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42045 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-02T21:14:43,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42045 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1 2024-12-02T21:14:43,615 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42045 {}] balancer.StochasticLoadBalancer(403): Cluster wide - Calculating plan. may take up to 30000ms to complete. 2024-12-02T21:14:43,616 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42045 {}] balancer.StochasticLoadBalancer(515): Start StochasticLoadBalancer.balancer, initial weighted average imbalance=0.06931633435746164, functionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.06415002990995843, need balance); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.0); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); computedMaxSteps=64800 2024-12-02T21:14:44,062 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42045 {}] balancer.StochasticLoadBalancer(562): Finished computing new moving plan. Computation took 448 ms to try 64800 different iterations. Found a solution that moves 1 regions; Going from a computed imbalance of 0.06931633435746164 to a new imbalance of 0.01690158032826896. funtionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.0); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.037037037037037035, need balance); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); 2024-12-02T21:14:44,064 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42045 {}] master.HMaster(2167): Balancer plans size is 1, the balance interval is 300000 ms, and the max number regions in transition is 27 2024-12-02T21:14:44,064 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42045 {}] master.HMaster(2172): balance hri=120cf3776e92a15d3276f11911c4042f, source=87c3fdb6c570,33459,1733174078298, destination=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:44,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42045 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test, region=120cf3776e92a15d3276f11911c4042f, REOPEN/MOVE 2024-12-02T21:14:44,066 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test, region=120cf3776e92a15d3276f11911c4042f, REOPEN/MOVE 2024-12-02T21:14:44,068 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=120cf3776e92a15d3276f11911c4042f, regionState=CLOSING, regionLocation=87c3fdb6c570,33459,1733174078298 2024-12-02T21:14:44,070 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=test, region=120cf3776e92a15d3276f11911c4042f, REOPEN/MOVE because future has completed 2024-12-02T21:14:44,070 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-02T21:14:44,071 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE, hasLock=false; CloseRegionProcedure 120cf3776e92a15d3276f11911c4042f, server=87c3fdb6c570,33459,1733174078298}] 2024-12-02T21:14:44,228 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(122): Close 120cf3776e92a15d3276f11911c4042f 2024-12-02T21:14:44,228 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-02T21:14:44,228 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1722): Closing 120cf3776e92a15d3276f11911c4042f, disabling compactions & flushes 2024-12-02T21:14:44,228 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1755): Closing region test,eee,1733174081291.120cf3776e92a15d3276f11911c4042f. 2024-12-02T21:14:44,228 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1733174081291.120cf3776e92a15d3276f11911c4042f. 2024-12-02T21:14:44,229 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1843): Acquired close lock on test,eee,1733174081291.120cf3776e92a15d3276f11911c4042f. after waiting 0 ms 2024-12-02T21:14:44,229 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1853): Updates disabled for region test,eee,1733174081291.120cf3776e92a15d3276f11911c4042f. 2024-12-02T21:14:44,234 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/120cf3776e92a15d3276f11911c4042f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T21:14:44,236 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1973): Closed test,eee,1733174081291.120cf3776e92a15d3276f11911c4042f. 2024-12-02T21:14:44,236 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1676): Region close journal for 120cf3776e92a15d3276f11911c4042f: Waiting for close lock at 1733174084228Running coprocessor pre-close hooks at 1733174084228Disabling compacts and flushes for region at 1733174084228Disabling writes for close at 1733174084229 (+1 ms)Writing region close event to WAL at 1733174084229Running coprocessor post-close hooks at 1733174084234 (+5 ms)Closed at 1733174084236 (+2 ms) 2024-12-02T21:14:44,237 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegionServer(3302): Adding 120cf3776e92a15d3276f11911c4042f move to 87c3fdb6c570,36585,1733174078182 record at close sequenceid=2 2024-12-02T21:14:44,240 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(157): Closed 120cf3776e92a15d3276f11911c4042f 2024-12-02T21:14:44,241 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=120cf3776e92a15d3276f11911c4042f, regionState=CLOSED 2024-12-02T21:14:44,243 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE, hasLock=false; CloseRegionProcedure 120cf3776e92a15d3276f11911c4042f, server=87c3fdb6c570,33459,1733174078298 because future has completed 2024-12-02T21:14:44,247 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-12-02T21:14:44,247 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; CloseRegionProcedure 120cf3776e92a15d3276f11911c4042f, server=87c3fdb6c570,33459,1733174078298 in 173 msec 2024-12-02T21:14:44,248 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=120cf3776e92a15d3276f11911c4042f, REOPEN/MOVE; state=CLOSED, location=87c3fdb6c570,36585,1733174078182; forceNewPlan=false, retain=false 2024-12-02T21:14:44,399 INFO [87c3fdb6c570:42045 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-02T21:14:44,399 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=120cf3776e92a15d3276f11911c4042f, regionState=OPENING, regionLocation=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:44,403 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=120cf3776e92a15d3276f11911c4042f, REOPEN/MOVE because future has completed 2024-12-02T21:14:44,404 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=57, state=RUNNABLE, hasLock=false; OpenRegionProcedure 120cf3776e92a15d3276f11911c4042f, server=87c3fdb6c570,36585,1733174078182}] 2024-12-02T21:14:44,563 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(132): Open test,eee,1733174081291.120cf3776e92a15d3276f11911c4042f. 2024-12-02T21:14:44,563 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7752): Opening region: {ENCODED => 120cf3776e92a15d3276f11911c4042f, NAME => 'test,eee,1733174081291.120cf3776e92a15d3276f11911c4042f.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-12-02T21:14:44,564 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 120cf3776e92a15d3276f11911c4042f 2024-12-02T21:14:44,564 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(898): Instantiated test,eee,1733174081291.120cf3776e92a15d3276f11911c4042f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:14:44,564 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7794): checking encryption for 120cf3776e92a15d3276f11911c4042f 2024-12-02T21:14:44,564 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7797): checking classloading for 120cf3776e92a15d3276f11911c4042f 2024-12-02T21:14:44,566 INFO [StoreOpener-120cf3776e92a15d3276f11911c4042f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 120cf3776e92a15d3276f11911c4042f 2024-12-02T21:14:44,568 INFO [StoreOpener-120cf3776e92a15d3276f11911c4042f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 120cf3776e92a15d3276f11911c4042f columnFamilyName family 2024-12-02T21:14:44,568 DEBUG [StoreOpener-120cf3776e92a15d3276f11911c4042f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:14:44,569 INFO [StoreOpener-120cf3776e92a15d3276f11911c4042f-1 {}] regionserver.HStore(327): Store=120cf3776e92a15d3276f11911c4042f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T21:14:44,569 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1038): replaying wal for 120cf3776e92a15d3276f11911c4042f 2024-12-02T21:14:44,570 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/120cf3776e92a15d3276f11911c4042f 2024-12-02T21:14:44,572 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/120cf3776e92a15d3276f11911c4042f 2024-12-02T21:14:44,573 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1048): stopping wal replay for 120cf3776e92a15d3276f11911c4042f 2024-12-02T21:14:44,573 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1060): Cleaning up temporary data for 120cf3776e92a15d3276f11911c4042f 2024-12-02T21:14:44,575 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1093): writing seq id for 120cf3776e92a15d3276f11911c4042f 2024-12-02T21:14:44,576 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1114): Opened 120cf3776e92a15d3276f11911c4042f; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65789394, jitterRate=-0.019661635160446167}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T21:14:44,576 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 120cf3776e92a15d3276f11911c4042f 2024-12-02T21:14:44,577 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1006): Region open journal for 120cf3776e92a15d3276f11911c4042f: Running coprocessor pre-open hook at 1733174084564Writing region info on filesystem at 1733174084564Initializing all the Stores at 1733174084566 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174084566Cleaning up temporary data from old regions at 1733174084573 (+7 ms)Running coprocessor post-open hooks at 1733174084577 (+4 ms)Region opened successfully at 1733174084577 2024-12-02T21:14:44,579 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1733174081291.120cf3776e92a15d3276f11911c4042f., pid=59, masterSystemTime=1733174084557 2024-12-02T21:14:44,583 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1733174081291.120cf3776e92a15d3276f11911c4042f. 2024-12-02T21:14:44,583 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(153): Opened test,eee,1733174081291.120cf3776e92a15d3276f11911c4042f. 2024-12-02T21:14:44,584 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=120cf3776e92a15d3276f11911c4042f, regionState=OPEN, openSeqNum=5, regionLocation=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:44,587 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=57, state=RUNNABLE, hasLock=false; OpenRegionProcedure 120cf3776e92a15d3276f11911c4042f, server=87c3fdb6c570,36585,1733174078182 because future has completed 2024-12-02T21:14:44,592 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=57 2024-12-02T21:14:44,592 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=57, state=SUCCESS, hasLock=false; OpenRegionProcedure 120cf3776e92a15d3276f11911c4042f, server=87c3fdb6c570,36585,1733174078182 in 184 msec 2024-12-02T21:14:44,597 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=120cf3776e92a15d3276f11911c4042f, REOPEN/MOVE in 528 msec 2024-12-02T21:14:44,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42045 {}] master.HMaster(2203): Balancer is going into sleep until next period in 300000ms 2024-12-02T21:14:44,670 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [900,000] milli-secs(wait.for.ratio=[1]) 2024-12-02T21:14:44,678 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42045 {}] master.MasterRpcServices(567): Client=jenkins//172.17.0.3 set balanceSwitch=false 2024-12-02T21:14:44,699 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.assignment.TestRollbackSCP#testFailAndRollback Thread=380, OpenFileDescriptor=641, MaxFileDescriptor=1048576, SystemLoadAverage=821, ProcessCount=11, AvailableMemoryMB=2880 2024-12-02T21:14:44,703 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table hbase:meta 2024-12-02T21:14:44,704 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=hbase:meta,,1 2024-12-02T21:14:44,704 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing 87c3fdb6c570,33459,1733174078298 2024-12-02T21:14:44,705 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 87c3fdb6c570,33459,1733174078298: Simulated kill ***** 2024-12-02T21:14:44,705 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-02T21:14:44,708 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-02T21:14:44,711 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-02T21:14:44,711 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-02T21:14:44,711 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-02T21:14:44,712 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 342459648 }, "NonHeapMemoryUsage": { "committed": 154009600, "init": 7667712, "max": -1, "used": 151650184 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "87c3fdb6c570" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "87c3fdb6c570" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.Context": "regionserver", "tag.Hostname": "87c3fdb6c570" } ] } 2024-12-02T21:14:44,715 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42045 {}] master.MasterRpcServices(700): 87c3fdb6c570,33459,1733174078298 reported a fatal error: ***** ABORTING region server 87c3fdb6c570,33459,1733174078298: Simulated kill ***** 2024-12-02T21:14:44,718 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '87c3fdb6c570,33459,1733174078298' ***** 2024-12-02T21:14:44,718 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-12-02T21:14:44,719 INFO [RS:2;87c3fdb6c570:33459 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-02T21:14:44,719 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [15,000] milli-secs(wait.for.ratio=[1]) 2024-12-02T21:14:44,719 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-02T21:14:44,719 INFO [RS:2;87c3fdb6c570:33459 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-12-02T21:14:44,719 INFO [RS:2;87c3fdb6c570:33459 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-12-02T21:14:44,719 INFO [RS:2;87c3fdb6c570:33459 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-02T21:14:44,719 INFO [RS:2;87c3fdb6c570:33459 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;87c3fdb6c570:33459. 2024-12-02T21:14:44,719 DEBUG [RS:2;87c3fdb6c570:33459 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T21:14:44,725 DEBUG [RS:2;87c3fdb6c570:33459 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:14:44,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741836_1012 (size=33749) 2024-12-02T21:14:44,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42827 is added to blk_1073741836_1012 (size=33749) 2024-12-02T21:14:44,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36203 is added to blk_1073741836_1012 (size=33749) 2024-12-02T21:14:44,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42827 is added to blk_1073741835_1011 (size=2774) 2024-12-02T21:14:44,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741835_1011 (size=2774) 2024-12-02T21:14:44,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36203 is added to blk_1073741835_1011 (size=2774) 2024-12-02T21:14:44,736 DEBUG [RS:2;87c3fdb6c570:33459 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:14:44,736 INFO [RS:2;87c3fdb6c570:33459 {}] regionserver.LeaseManager(133): Closed leases 2024-12-02T21:14:44,736 INFO [RS:2;87c3fdb6c570:33459 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:33459 2024-12-02T21:14:44,756 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33459-0x10197e5a0060003, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/87c3fdb6c570,33459,1733174078298 2024-12-02T21:14:44,756 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42045-0x10197e5a0060000, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-02T21:14:44,756 INFO [RS:2;87c3fdb6c570:33459 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-02T21:14:44,758 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [87c3fdb6c570,33459,1733174078298] 2024-12-02T21:14:44,777 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/87c3fdb6c570,33459,1733174078298 already deleted, retry=false 2024-12-02T21:14:44,778 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 87c3fdb6c570,33459,1733174078298 on 87c3fdb6c570,42045,1733174077295 2024-12-02T21:14:44,783 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true 2024-12-02T21:14:44,787 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=60 for 87c3fdb6c570,33459,1733174078298 (carryingMeta=true) 87c3fdb6c570,33459,1733174078298/CRASHED/regionCount=9/lock=java.util.concurrent.locks.ReentrantReadWriteLock@68668a30[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-02T21:14:44,788 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=60, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true 2024-12-02T21:14:44,790 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=60, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true, isMeta: true 2024-12-02T21:14:44,792 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/WALs/87c3fdb6c570,33459,1733174078298-splitting 2024-12-02T21:14:44,794 INFO [PEWorker-1 {}] master.SplitWALManager(105): 87c3fdb6c570,33459,1733174078298 WAL count=1, meta=true 2024-12-02T21:14:44,796 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta}] 2024-12-02T21:14:44,800 INFO [regionserver/87c3fdb6c570:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-02T21:14:44,802 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:44,806 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061}] 2024-12-02T21:14:44,820 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-12-02T21:14:44,867 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33459-0x10197e5a0060003, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T21:14:44,867 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33459-0x10197e5a0060003, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T21:14:44,868 INFO [RS:2;87c3fdb6c570:33459 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-02T21:14:44,868 INFO [RS:2;87c3fdb6c570:33459 {}] regionserver.HRegionServer(1031): Exiting; stopping=87c3fdb6c570,33459,1733174078298; zookeeper connection closed. 2024-12-02T21:14:44,868 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@49b5bf1d {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@49b5bf1d 2024-12-02T21:14:44,968 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=62 2024-12-02T21:14:44,988 INFO [RS_LOG_REPLAY_OPS-regionserver/87c3fdb6c570:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(299): Splitting hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/WALs/87c3fdb6c570,33459,1733174078298-splitting/87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, size=33.0 K (33749bytes) 2024-12-02T21:14:44,988 DEBUG [RS_LOG_REPLAY_OPS-regionserver/87c3fdb6c570:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-12-02T21:14:44,989 DEBUG [RS_LOG_REPLAY_OPS-regionserver/87c3fdb6c570:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-12-02T21:14:44,989 INFO [RS_LOG_REPLAY_OPS-regionserver/87c3fdb6c570:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/WALs/87c3fdb6c570,33459,1733174078298-splitting/87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta 2024-12-02T21:14:44,994 INFO [RS_LOG_REPLAY_OPS-regionserver/87c3fdb6c570:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/WALs/87c3fdb6c570,33459,1733174078298-splitting/87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta after 3ms 2024-12-02T21:14:45,002 INFO [RS_LOG_REPLAY_OPS-regionserver/87c3fdb6c570:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(310): Open hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/WALs/87c3fdb6c570,33459,1733174078298-splitting/87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta took 16ms 2024-12-02T21:14:45,010 DEBUG [RS_LOG_REPLAY_OPS-regionserver/87c3fdb6c570:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-12-02T21:14:45,018 INFO [RS_LOG_REPLAY_OPS-regionserver/87c3fdb6c570:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta.temp 2024-12-02T21:14:45,019 DEBUG [RS_LOG_REPLAY_OPS-regionserver/87c3fdb6c570:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/WALs/87c3fdb6c570,33459,1733174078298-splitting/87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta so closing down 2024-12-02T21:14:45,019 DEBUG [RS_LOG_REPLAY_OPS-regionserver/87c3fdb6c570:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-02T21:14:45,022 INFO [RS_LOG_REPLAY_OPS-regionserver/87c3fdb6c570:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta.temp 2024-12-02T21:14:45,025 INFO [RS_LOG_REPLAY_OPS-regionserver/87c3fdb6c570:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.OutputSink(145): 3 split writer threads finished 2024-12-02T21:14:45,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741864_1040 (size=33508) 2024-12-02T21:14:45,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36203 is added to blk_1073741864_1040 (size=33508) 2024-12-02T21:14:45,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42827 is added to blk_1073741864_1040 (size=33508) 2024-12-02T21:14:45,035 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta.temp (wrote 61 edits, skipped 0 edits in 2 ms) 2024-12-02T21:14:45,037 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta.temp to hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-12-02T21:14:45,041 INFO [RS_LOG_REPLAY_OPS-regionserver/87c3fdb6c570:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(425): Processed 62 edits across 1 Regions in 33 ms; skipped=1; WAL=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/WALs/87c3fdb6c570,33459,1733174078298-splitting/87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, size=33.0 K, length=33749, corrupted=false, cancelled=false 2024-12-02T21:14:45,041 DEBUG [RS_LOG_REPLAY_OPS-regionserver/87c3fdb6c570:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(428): Completed split of hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/WALs/87c3fdb6c570,33459,1733174078298-splitting/87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, journal: Splitting hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/WALs/87c3fdb6c570,33459,1733174078298-splitting/87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, size=33.0 K (33749bytes) at 1733174084988Finishing writing output for hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/WALs/87c3fdb6c570,33459,1733174078298-splitting/87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta so closing down at 1733174085019 (+31 ms)Creating recovered edits writer path=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta.temp at 1733174085022 (+3 ms)3 split writer threads finished at 1733174085025 (+3 ms)Closed recovered edits writer path=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta.temp (wrote 61 edits, skipped 0 edits in 2 ms) at 1733174085035 (+10 ms)Rename recovered edits hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta.temp to hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 at 1733174085037 (+2 ms)Processed 62 edits across 1 Regions in 33 ms; skipped=1; WAL=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/WALs/87c3fdb6c570,33459,1733174078298-splitting/87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, size=33.0 K, length=33749, corrupted=false, cancelled=false at 1733174085041 (+4 ms) 2024-12-02T21:14:45,041 DEBUG [RS_LOG_REPLAY_OPS-regionserver/87c3fdb6c570:0-0 {event_type=RS_LOG_REPLAY, pid=62}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/WALs/87c3fdb6c570,33459,1733174078298-splitting/87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta 2024-12-02T21:14:45,042 DEBUG [RS_LOG_REPLAY_OPS-regionserver/87c3fdb6c570:0-0 {event_type=RS_LOG_REPLAY, pid=62}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=62 2024-12-02T21:14:45,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42045 {}] master.HMaster(4169): Remote procedure done, pid=62 2024-12-02T21:14:45,052 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/WALs/87c3fdb6c570,33459,1733174078298-splitting/87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta to hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/oldWALs 2024-12-02T21:14:45,056 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-12-02T21:14:45,056 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 in 247 msec 2024-12-02T21:14:45,058 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:45,062 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-12-02T21:14:45,062 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 in 263 msec 2024-12-02T21:14:45,064 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/WALs/87c3fdb6c570,33459,1733174078298-splitting dir is empty, no logs to split. 2024-12-02T21:14:45,065 INFO [PEWorker-1 {}] master.SplitWALManager(105): 87c3fdb6c570,33459,1733174078298 WAL count=0, meta=true 2024-12-02T21:14:45,065 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 87c3fdb6c570,33459,1733174078298 WAL splitting is done? wals=0, meta=true 2024-12-02T21:14:45,067 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-02T21:14:45,069 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T21:14:45,070 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-02T21:14:45,221 DEBUG [87c3fdb6c570:42045 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-02T21:14:45,221 DEBUG [87c3fdb6c570:42045 {}] balancer.BalancerClusterState(204): Hosts are {87c3fdb6c570=0} racks are {/default-rack=0} 2024-12-02T21:14:45,222 DEBUG [87c3fdb6c570:42045 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-02T21:14:45,222 DEBUG [87c3fdb6c570:42045 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-02T21:14:45,222 DEBUG [87c3fdb6c570:42045 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-02T21:14:45,222 DEBUG [87c3fdb6c570:42045 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-02T21:14:45,222 INFO [87c3fdb6c570:42045 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-02T21:14:45,222 INFO [87c3fdb6c570:42045 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-02T21:14:45,222 DEBUG [87c3fdb6c570:42045 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-02T21:14:45,223 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=63 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:45,226 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 87c3fdb6c570,40413,1733174078061, state=OPENING 2024-12-02T21:14:45,234 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40413-0x10197e5a0060001, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:45,234 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42045-0x10197e5a0060000, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:45,234 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36585-0x10197e5a0060002, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:45,234 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T21:14:45,234 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:45,234 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=87c3fdb6c570,40413,1733174078061}] 2024-12-02T21:14:45,234 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:45,235 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:45,293 INFO [regionserver/87c3fdb6c570:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-02T21:14:45,294 INFO [regionserver/87c3fdb6c570:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-02T21:14:45,392 INFO [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=64}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-02T21:14:45,392 INFO [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T21:14:45,393 INFO [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=64}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-02T21:14:45,395 INFO [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=87c3fdb6c570%2C40413%2C1733174078061.meta, suffix=.meta, logDir=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/WALs/87c3fdb6c570,40413,1733174078061, archiveDir=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/oldWALs, maxLogs=32 2024-12-02T21:14:45,410 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=64}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/WALs/87c3fdb6c570,40413,1733174078061/87c3fdb6c570%2C40413%2C1733174078061.meta.1733174085396.meta, exclude list is [], retry=0 2024-12-02T21:14:45,414 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36203,DS-2d489d52-306c-448f-8298-290ac4052947,DISK] 2024-12-02T21:14:45,414 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42827,DS-9aa11248-a7e5-4660-b440-2c286bf86b1b,DISK] 2024-12-02T21:14:45,414 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40959,DS-54bbf138-7100-4d62-a6b6-08df6ef7aa28,DISK] 2024-12-02T21:14:45,417 INFO [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/WALs/87c3fdb6c570,40413,1733174078061/87c3fdb6c570%2C40413%2C1733174078061.meta.1733174085396.meta 2024-12-02T21:14:45,417 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34107:34107),(127.0.0.1/127.0.0.1:46475:46475),(127.0.0.1/127.0.0.1:35813:35813)] 2024-12-02T21:14:45,417 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-02T21:14:45,418 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=64}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-02T21:14:45,418 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-02T21:14:45,418 INFO [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-02T21:14:45,418 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-02T21:14:45,418 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:14:45,418 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-02T21:14:45,418 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-02T21:14:45,420 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-02T21:14:45,421 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-02T21:14:45,421 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:14:45,421 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T21:14:45,422 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-02T21:14:45,423 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-02T21:14:45,423 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:14:45,423 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T21:14:45,423 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-02T21:14:45,424 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-02T21:14:45,424 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:14:45,425 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T21:14:45,425 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-02T21:14:45,426 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-02T21:14:45,426 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:14:45,427 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T21:14:45,427 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-02T21:14:45,428 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/hbase/meta/1588230740 2024-12-02T21:14:45,431 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/hbase/meta/1588230740 2024-12-02T21:14:45,432 INFO [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-12-02T21:14:45,447 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5793): Applied 277, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=64, path=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-12-02T21:14:45,450 INFO [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=29.41 KB heapSize=50.67 KB 2024-12-02T21:14:45,510 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/hbase/meta/1588230740/.tmp/info/693974fda0934637825ddc68efb94345 is 138, key is test,bbb,1733174081291.5ed86f04f519cc5dccd1b6d41be3da7d./info:regioninfo/1733174082849/Put/seqid=0 2024-12-02T21:14:45,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36203 is added to blk_1073741866_1042 (size=36385) 2024-12-02T21:14:45,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42827 is added to blk_1073741866_1042 (size=36385) 2024-12-02T21:14:45,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741866_1042 (size=36385) 2024-12-02T21:14:45,526 INFO [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.26 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/hbase/meta/1588230740/.tmp/info/693974fda0934637825ddc68efb94345 2024-12-02T21:14:45,595 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/hbase/meta/1588230740/.tmp/ns/ea5fb74a6f5d41f7876065730aeee3c1 is 43, key is default/ns:d/1733174081073/Put/seqid=0 2024-12-02T21:14:45,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42827 is added to blk_1073741867_1043 (size=5153) 2024-12-02T21:14:45,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36203 is added to blk_1073741867_1043 (size=5153) 2024-12-02T21:14:45,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741867_1043 (size=5153) 2024-12-02T21:14:45,604 INFO [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/hbase/meta/1588230740/.tmp/ns/ea5fb74a6f5d41f7876065730aeee3c1 2024-12-02T21:14:45,633 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/hbase/meta/1588230740/.tmp/table/2c139b99897a41e298ed9fc526b86673 is 40, key is test/table:state/1733174082924/Put/seqid=0 2024-12-02T21:14:45,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36203 is added to blk_1073741868_1044 (size=5165) 2024-12-02T21:14:45,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42827 is added to blk_1073741868_1044 (size=5165) 2024-12-02T21:14:45,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741868_1044 (size=5165) 2024-12-02T21:14:46,058 INFO [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=72 B at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/hbase/meta/1588230740/.tmp/table/2c139b99897a41e298ed9fc526b86673 2024-12-02T21:14:46,070 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/hbase/meta/1588230740/.tmp/info/693974fda0934637825ddc68efb94345 as hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/hbase/meta/1588230740/info/693974fda0934637825ddc68efb94345 2024-12-02T21:14:46,080 INFO [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/hbase/meta/1588230740/info/693974fda0934637825ddc68efb94345, entries=273, sequenceid=64, filesize=35.5 K 2024-12-02T21:14:46,082 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/hbase/meta/1588230740/.tmp/ns/ea5fb74a6f5d41f7876065730aeee3c1 as hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/hbase/meta/1588230740/ns/ea5fb74a6f5d41f7876065730aeee3c1 2024-12-02T21:14:46,091 INFO [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/hbase/meta/1588230740/ns/ea5fb74a6f5d41f7876065730aeee3c1, entries=2, sequenceid=64, filesize=5.0 K 2024-12-02T21:14:46,092 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/hbase/meta/1588230740/.tmp/table/2c139b99897a41e298ed9fc526b86673 as hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/hbase/meta/1588230740/table/2c139b99897a41e298ed9fc526b86673 2024-12-02T21:14:46,099 INFO [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/hbase/meta/1588230740/table/2c139b99897a41e298ed9fc526b86673, entries=2, sequenceid=64, filesize=5.0 K 2024-12-02T21:14:46,104 INFO [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(3140): Finished flush of dataSize ~29.41 KB/30111, heapSize ~50.38 KB/51584, currentSize=0 B/0 for 1588230740 in 651ms, sequenceid=64, compaction requested=false; wal=null 2024-12-02T21:14:46,105 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-02T21:14:46,108 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-12-02T21:14:46,109 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-02T21:14:46,110 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-02T21:14:46,110 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-02T21:14:46,112 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-02T21:14:46,115 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/hbase/meta/1588230740/recovered.edits/64.seqid, newMaxSeqId=64, maxSeqId=1 2024-12-02T21:14:46,116 INFO [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=65; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62857598, jitterRate=-0.06334879994392395}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-02T21:14:46,116 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-02T21:14:46,116 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733174085418Writing region info on filesystem at 1733174085418Initializing all the Stores at 1733174085419 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733174085419Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733174085420 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174085420Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733174085420Obtaining lock to block concurrent updates at 1733174085450 (+30 ms)Preparing flush snapshotting stores in 1588230740 at 1733174085450Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=30111, getHeapSize=51824, getOffHeapSize=0, getCellsCount=277 at 1733174085453 (+3 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733174085453Flushing 1588230740/info: creating writer at 1733174085454 (+1 ms)Flushing 1588230740/info: appending metadata at 1733174085498 (+44 ms)Flushing 1588230740/info: closing flushed file at 1733174085501 (+3 ms)Flushing 1588230740/ns: creating writer at 1733174085578 (+77 ms)Flushing 1588230740/ns: appending metadata at 1733174085594 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1733174085594Flushing 1588230740/table: creating writer at 1733174085611 (+17 ms)Flushing 1588230740/table: appending metadata at 1733174085632 (+21 ms)Flushing 1588230740/table: closing flushed file at 1733174085632Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6b16de25: reopening flushed file at 1733174086068 (+436 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d275920: reopening flushed file at 1733174086081 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@34d71977: reopening flushed file at 1733174086091 (+10 ms)Finished flush of dataSize ~29.41 KB/30111, heapSize ~50.38 KB/51584, currentSize=0 B/0 for 1588230740 in 651ms, sequenceid=64, compaction requested=false; wal=null at 1733174086104 (+13 ms)Cleaning up temporary data from old regions at 1733174086110 (+6 ms)Running coprocessor post-open hooks at 1733174086116 (+6 ms)Region opened successfully at 1733174086116 2024-12-02T21:14:46,118 INFO [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=64, masterSystemTime=1733174085387 2024-12-02T21:14:46,120 DEBUG [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-02T21:14:46,120 INFO [RS_OPEN_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_META, pid=64}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-02T21:14:46,121 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=63 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=65, regionLocation=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:46,122 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 87c3fdb6c570,40413,1733174078061, state=OPEN 2024-12-02T21:14:46,135 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40413-0x10197e5a0060001, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:46,135 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36585-0x10197e5a0060002, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:46,135 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42045-0x10197e5a0060000, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:46,135 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:46,135 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:46,135 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:46,135 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=64, ppid=63, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:46,138 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-12-02T21:14:46,139 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=87c3fdb6c570,40413,1733174078061 in 901 msec 2024-12-02T21:14:46,142 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=60 2024-12-02T21:14:46,142 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1.0720 sec 2024-12-02T21:14:46,143 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 87c3fdb6c570,33459,1733174078298 had 9 regions 2024-12-02T21:14:46,144 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=60, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true, isMeta: false 2024-12-02T21:14:46,147 INFO [PEWorker-1 {}] master.SplitWALManager(105): 87c3fdb6c570,33459,1733174078298 WAL count=1, meta=false 2024-12-02T21:14:46,147 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=60, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466}] 2024-12-02T21:14:46,149 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:46,150 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182}] 2024-12-02T21:14:46,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36585 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=66 2024-12-02T21:14:46,319 INFO [RS_LOG_REPLAY_OPS-regionserver/87c3fdb6c570:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(299): Splitting hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/WALs/87c3fdb6c570,33459,1733174078298-splitting/87c3fdb6c570%2C33459%2C1733174078298.1733174080466, size=2.7 K (2774bytes) 2024-12-02T21:14:46,319 INFO [RS_LOG_REPLAY_OPS-regionserver/87c3fdb6c570:0-0 {event_type=RS_LOG_REPLAY, pid=66}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/WALs/87c3fdb6c570,33459,1733174078298-splitting/87c3fdb6c570%2C33459%2C1733174078298.1733174080466 2024-12-02T21:14:46,319 INFO [RS_LOG_REPLAY_OPS-regionserver/87c3fdb6c570:0-0 {event_type=RS_LOG_REPLAY, pid=66}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/WALs/87c3fdb6c570,33459,1733174078298-splitting/87c3fdb6c570%2C33459%2C1733174078298.1733174080466 after 0ms 2024-12-02T21:14:46,322 INFO [RS_LOG_REPLAY_OPS-regionserver/87c3fdb6c570:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(310): Open hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/WALs/87c3fdb6c570,33459,1733174078298-splitting/87c3fdb6c570%2C33459%2C1733174078298.1733174080466 took 4ms 2024-12-02T21:14:46,326 DEBUG [RS_LOG_REPLAY_OPS-regionserver/87c3fdb6c570:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 06d131e39cb4f5b02db4ec472d438a26: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-02T21:14:46,327 DEBUG [RS_LOG_REPLAY_OPS-regionserver/87c3fdb6c570:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 0d0be65f6ffd7ebb4c8b2fee37042136: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-02T21:14:46,329 DEBUG [RS_LOG_REPLAY_OPS-regionserver/87c3fdb6c570:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for d63da23e8236e18b416716ec838dd43e: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-02T21:14:46,331 DEBUG [RS_LOG_REPLAY_OPS-regionserver/87c3fdb6c570:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 1a4ea0dec946dcfe96ad162d5bcf5949: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-02T21:14:46,333 DEBUG [RS_LOG_REPLAY_OPS-regionserver/87c3fdb6c570:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 87ba365a3e5d028c9193155af96aeb60: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-02T21:14:46,335 DEBUG [RS_LOG_REPLAY_OPS-regionserver/87c3fdb6c570:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for a50bae1d5cb3d03a86dc103b09a0f68b: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-02T21:14:46,338 DEBUG [RS_LOG_REPLAY_OPS-regionserver/87c3fdb6c570:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 0dfd4169e4c254c51bcc70a472f99ecf: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-02T21:14:46,339 DEBUG [RS_LOG_REPLAY_OPS-regionserver/87c3fdb6c570:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 0517912ff18e81317bb5474cefaca025: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-02T21:14:46,341 DEBUG [RS_LOG_REPLAY_OPS-regionserver/87c3fdb6c570:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 120cf3776e92a15d3276f11911c4042f: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-02T21:14:46,341 DEBUG [RS_LOG_REPLAY_OPS-regionserver/87c3fdb6c570:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/WALs/87c3fdb6c570,33459,1733174078298-splitting/87c3fdb6c570%2C33459%2C1733174078298.1733174080466 so closing down 2024-12-02T21:14:46,341 DEBUG [RS_LOG_REPLAY_OPS-regionserver/87c3fdb6c570:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-02T21:14:46,342 INFO [RS_LOG_REPLAY_OPS-regionserver/87c3fdb6c570:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.OutputSink(145): 3 split writer threads finished 2024-12-02T21:14:46,342 INFO [RS_LOG_REPLAY_OPS-regionserver/87c3fdb6c570:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(425): Processed 10 edits across 0 Regions in 19 ms; skipped=10; WAL=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/WALs/87c3fdb6c570,33459,1733174078298-splitting/87c3fdb6c570%2C33459%2C1733174078298.1733174080466, size=2.7 K, length=2774, corrupted=false, cancelled=false 2024-12-02T21:14:46,342 DEBUG [RS_LOG_REPLAY_OPS-regionserver/87c3fdb6c570:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(428): Completed split of hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/WALs/87c3fdb6c570,33459,1733174078298-splitting/87c3fdb6c570%2C33459%2C1733174078298.1733174080466, journal: Splitting hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/WALs/87c3fdb6c570,33459,1733174078298-splitting/87c3fdb6c570%2C33459%2C1733174078298.1733174080466, size=2.7 K (2774bytes) at 1733174086319Finishing writing output for hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/WALs/87c3fdb6c570,33459,1733174078298-splitting/87c3fdb6c570%2C33459%2C1733174078298.1733174080466 so closing down at 1733174086341 (+22 ms)3 split writer threads finished at 1733174086342 (+1 ms)Processed 10 edits across 0 Regions in 19 ms; skipped=10; WAL=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/WALs/87c3fdb6c570,33459,1733174078298-splitting/87c3fdb6c570%2C33459%2C1733174078298.1733174080466, size=2.7 K, length=2774, corrupted=false, cancelled=false at 1733174086342 2024-12-02T21:14:46,342 DEBUG [RS_LOG_REPLAY_OPS-regionserver/87c3fdb6c570:0-0 {event_type=RS_LOG_REPLAY, pid=66}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/WALs/87c3fdb6c570,33459,1733174078298-splitting/87c3fdb6c570%2C33459%2C1733174078298.1733174080466 2024-12-02T21:14:46,342 DEBUG [RS_LOG_REPLAY_OPS-regionserver/87c3fdb6c570:0-0 {event_type=RS_LOG_REPLAY, pid=66}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=66 2024-12-02T21:14:46,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42045 {}] master.HMaster(4169): Remote procedure done, pid=66 2024-12-02T21:14:46,347 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/WALs/87c3fdb6c570,33459,1733174078298-splitting/87c3fdb6c570%2C33459%2C1733174078298.1733174080466 to hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/oldWALs 2024-12-02T21:14:46,349 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=66, resume processing ppid=65 2024-12-02T21:14:46,349 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 in 197 msec 2024-12-02T21:14:46,351 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:46,353 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=60 2024-12-02T21:14:46,353 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 in 204 msec 2024-12-02T21:14:46,355 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/WALs/87c3fdb6c570,33459,1733174078298-splitting dir is empty, no logs to split. 2024-12-02T21:14:46,355 INFO [PEWorker-1 {}] master.SplitWALManager(105): 87c3fdb6c570,33459,1733174078298 WAL count=0, meta=false 2024-12-02T21:14:46,355 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 87c3fdb6c570,33459,1733174078298 WAL splitting is done? wals=0, meta=false 2024-12-02T21:14:46,357 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN}, {pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN}, {pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN}, {pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=87ba365a3e5d028c9193155af96aeb60, ASSIGN}, {pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b, ASSIGN}, {pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0dfd4169e4c254c51bcc70a472f99ecf, ASSIGN}, {pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d63da23e8236e18b416716ec838dd43e, ASSIGN}, {pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0d0be65f6ffd7ebb4c8b2fee37042136, ASSIGN}] 2024-12-02T21:14:46,359 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0d0be65f6ffd7ebb4c8b2fee37042136, ASSIGN 2024-12-02T21:14:46,360 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=0d0be65f6ffd7ebb4c8b2fee37042136, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-02T21:14:46,361 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d63da23e8236e18b416716ec838dd43e, ASSIGN 2024-12-02T21:14:46,362 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d63da23e8236e18b416716ec838dd43e, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-02T21:14:46,364 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0dfd4169e4c254c51bcc70a472f99ecf, ASSIGN 2024-12-02T21:14:46,365 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=0dfd4169e4c254c51bcc70a472f99ecf, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-02T21:14:46,366 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b, ASSIGN 2024-12-02T21:14:46,367 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-02T21:14:46,368 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=87ba365a3e5d028c9193155af96aeb60, ASSIGN 2024-12-02T21:14:46,369 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=87ba365a3e5d028c9193155af96aeb60, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-02T21:14:46,369 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN 2024-12-02T21:14:46,370 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-02T21:14:46,371 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN 2024-12-02T21:14:46,372 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-02T21:14:46,373 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN 2024-12-02T21:14:46,374 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-02T21:14:46,410 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-02T21:14:46,468 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test' 2024-12-02T21:14:46,522 DEBUG [87c3fdb6c570:42045 {}] balancer.BalancerClusterState(204): Hosts are {87c3fdb6c570=0} racks are {/default-rack=0} 2024-12-02T21:14:46,522 DEBUG [87c3fdb6c570:42045 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-02T21:14:46,522 DEBUG [87c3fdb6c570:42045 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-02T21:14:46,522 DEBUG [87c3fdb6c570:42045 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-02T21:14:46,522 DEBUG [87c3fdb6c570:42045 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-02T21:14:46,522 INFO [87c3fdb6c570:42045 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-02T21:14:46,522 INFO [87c3fdb6c570:42045 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-02T21:14:46,522 DEBUG [87c3fdb6c570:42045 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-02T21:14:46,523 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=1a4ea0dec946dcfe96ad162d5bcf5949, regionState=OPENING, regionLocation=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:46,524 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=a50bae1d5cb3d03a86dc103b09a0f68b, regionState=OPENING, regionLocation=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:46,525 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=74 updating hbase:meta row=0d0be65f6ffd7ebb4c8b2fee37042136, regionState=OPENING, regionLocation=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:46,525 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=0517912ff18e81317bb5474cefaca025, regionState=OPENING, regionLocation=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:46,526 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=06d131e39cb4f5b02db4ec472d438a26, regionState=OPENING, regionLocation=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:46,527 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=73 updating hbase:meta row=d63da23e8236e18b416716ec838dd43e, regionState=OPENING, regionLocation=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:46,527 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=87ba365a3e5d028c9193155af96aeb60, regionState=OPENING, regionLocation=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:46,529 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 87c3fdb6c570:33459 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 87c3fdb6c570/172.17.0.3:33459 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:14:46,531 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,33459,1733174078298, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,33459,1733174078298, seqNum=-1, error=java.net.ConnectException: Call to address=87c3fdb6c570:33459 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 87c3fdb6c570/172.17.0.3:33459 2024-12-02T21:14:46,532 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,33459,1733174078298, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-12-02T21:14:46,532 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,33459,1733174078298, seqNum=-1 from cache 2024-12-02T21:14:46,533 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,33459,1733174078298, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=87c3fdb6c570:33459 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 87c3fdb6c570/172.17.0.3:33459 2024-12-02T21:14:46,533 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,33459,1733174078298, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=87c3fdb6c570:33459 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 87c3fdb6c570/172.17.0.3:33459 2024-12-02T21:14:46,533 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,33459,1733174078298, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=87c3fdb6c570:33459 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 87c3fdb6c570/172.17.0.3:33459 2024-12-02T21:14:46,533 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,33459,1733174078298, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=87c3fdb6c570:33459 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 87c3fdb6c570/172.17.0.3:33459 2024-12-02T21:14:46,534 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,33459,1733174078298, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=87c3fdb6c570:33459 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 87c3fdb6c570/172.17.0.3:33459 2024-12-02T21:14:46,534 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,33459,1733174078298, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=87c3fdb6c570:33459 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 87c3fdb6c570/172.17.0.3:33459 2024-12-02T21:14:46,534 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address 87c3fdb6c570:33459 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 87c3fdb6c570/172.17.0.3:33459 2024-12-02T21:14:46,536 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=72 updating hbase:meta row=0dfd4169e4c254c51bcc70a472f99ecf, regionState=OPENING, regionLocation=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:46,536 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T21:14:46,536 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=87c3fdb6c570,40413,1733174078061, seqNum=-1] 2024-12-02T21:14:46,537 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T21:14:46,540 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37765, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T21:14:46,544 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=0dfd4169e4c254c51bcc70a472f99ecf, ASSIGN because future has completed 2024-12-02T21:14:46,546 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0dfd4169e4c254c51bcc70a472f99ecf, server=87c3fdb6c570,36585,1733174078182}] 2024-12-02T21:14:46,646 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=87ba365a3e5d028c9193155af96aeb60, ASSIGN because future has completed 2024-12-02T21:14:46,646 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 87ba365a3e5d028c9193155af96aeb60, server=87c3fdb6c570,36585,1733174078182}] 2024-12-02T21:14:46,649 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN because future has completed 2024-12-02T21:14:46,653 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b, ASSIGN because future has completed 2024-12-02T21:14:46,654 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=0d0be65f6ffd7ebb4c8b2fee37042136, ASSIGN because future has completed 2024-12-02T21:14:46,655 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0d0be65f6ffd7ebb4c8b2fee37042136, server=87c3fdb6c570,40413,1733174078061}] 2024-12-02T21:14:46,657 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN because future has completed 2024-12-02T21:14:46,662 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN because future has completed 2024-12-02T21:14:46,664 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d63da23e8236e18b416716ec838dd43e, ASSIGN because future has completed 2024-12-02T21:14:46,666 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182}] 2024-12-02T21:14:46,674 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06d131e39cb4f5b02db4ec472d438a26, server=87c3fdb6c570,36585,1733174078182}] 2024-12-02T21:14:46,681 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0517912ff18e81317bb5474cefaca025, server=87c3fdb6c570,40413,1733174078061}] 2024-12-02T21:14:46,687 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure a50bae1d5cb3d03a86dc103b09a0f68b, server=87c3fdb6c570,40413,1733174078061}] 2024-12-02T21:14:46,696 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1a4ea0dec946dcfe96ad162d5bcf5949, server=87c3fdb6c570,40413,1733174078061}] 2024-12-02T21:14:46,715 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(132): Open test,ttt,1733174081291.d63da23e8236e18b416716ec838dd43e. 2024-12-02T21:14:46,715 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7752): Opening region: {ENCODED => d63da23e8236e18b416716ec838dd43e, NAME => 'test,ttt,1733174081291.d63da23e8236e18b416716ec838dd43e.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-12-02T21:14:46,715 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d63da23e8236e18b416716ec838dd43e 2024-12-02T21:14:46,716 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(898): Instantiated test,ttt,1733174081291.d63da23e8236e18b416716ec838dd43e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:14:46,716 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7794): checking encryption for d63da23e8236e18b416716ec838dd43e 2024-12-02T21:14:46,716 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7797): checking classloading for d63da23e8236e18b416716ec838dd43e 2024-12-02T21:14:46,718 INFO [StoreOpener-d63da23e8236e18b416716ec838dd43e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d63da23e8236e18b416716ec838dd43e 2024-12-02T21:14:46,720 INFO [StoreOpener-d63da23e8236e18b416716ec838dd43e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d63da23e8236e18b416716ec838dd43e columnFamilyName family 2024-12-02T21:14:46,720 DEBUG [StoreOpener-d63da23e8236e18b416716ec838dd43e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:14:46,721 INFO [StoreOpener-d63da23e8236e18b416716ec838dd43e-1 {}] regionserver.HStore(327): Store=d63da23e8236e18b416716ec838dd43e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T21:14:46,722 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1038): replaying wal for d63da23e8236e18b416716ec838dd43e 2024-12-02T21:14:46,723 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/d63da23e8236e18b416716ec838dd43e 2024-12-02T21:14:46,725 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/d63da23e8236e18b416716ec838dd43e 2024-12-02T21:14:46,726 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1048): stopping wal replay for d63da23e8236e18b416716ec838dd43e 2024-12-02T21:14:46,726 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1060): Cleaning up temporary data for d63da23e8236e18b416716ec838dd43e 2024-12-02T21:14:46,729 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1093): writing seq id for d63da23e8236e18b416716ec838dd43e 2024-12-02T21:14:46,731 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1114): Opened d63da23e8236e18b416716ec838dd43e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61819688, jitterRate=-0.07881486415863037}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T21:14:46,731 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d63da23e8236e18b416716ec838dd43e 2024-12-02T21:14:46,731 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1006): Region open journal for d63da23e8236e18b416716ec838dd43e: Running coprocessor pre-open hook at 1733174086716Writing region info on filesystem at 1733174086716Initializing all the Stores at 1733174086717 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174086717Cleaning up temporary data from old regions at 1733174086726 (+9 ms)Running coprocessor post-open hooks at 1733174086731 (+5 ms)Region opened successfully at 1733174086731 2024-12-02T21:14:46,733 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1733174081291.d63da23e8236e18b416716ec838dd43e., pid=78, masterSystemTime=1733174086708 2024-12-02T21:14:46,737 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1733174081291.d63da23e8236e18b416716ec838dd43e. 2024-12-02T21:14:46,737 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(153): Opened test,ttt,1733174081291.d63da23e8236e18b416716ec838dd43e. 2024-12-02T21:14:46,737 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(132): Open test,nnn,1733174081291.87ba365a3e5d028c9193155af96aeb60. 2024-12-02T21:14:46,738 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7752): Opening region: {ENCODED => 87ba365a3e5d028c9193155af96aeb60, NAME => 'test,nnn,1733174081291.87ba365a3e5d028c9193155af96aeb60.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-12-02T21:14:46,738 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 87ba365a3e5d028c9193155af96aeb60 2024-12-02T21:14:46,738 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(898): Instantiated test,nnn,1733174081291.87ba365a3e5d028c9193155af96aeb60.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:14:46,738 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7794): checking encryption for 87ba365a3e5d028c9193155af96aeb60 2024-12-02T21:14:46,739 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7797): checking classloading for 87ba365a3e5d028c9193155af96aeb60 2024-12-02T21:14:46,740 WARN [PEWorker-1 {}] procedure2.ProcedureTestingUtility(201): Set Kill before store update in rollback to: true 2024-12-02T21:14:46,741 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:46,741 INFO [StoreOpener-87ba365a3e5d028c9193155af96aeb60-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 87ba365a3e5d028c9193155af96aeb60 2024-12-02T21:14:46,743 INFO [StoreOpener-87ba365a3e5d028c9193155af96aeb60-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 87ba365a3e5d028c9193155af96aeb60 columnFamilyName family 2024-12-02T21:14:46,741 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1844): CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182 java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:14:46,743 DEBUG [StoreOpener-87ba365a3e5d028c9193155af96aeb60-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:14:46,744 INFO [StoreOpener-87ba365a3e5d028c9193155af96aeb60-1 {}] regionserver.HStore(327): Store=87ba365a3e5d028c9193155af96aeb60/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T21:14:46,744 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1038): replaying wal for 87ba365a3e5d028c9193155af96aeb60 2024-12-02T21:14:46,745 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/87ba365a3e5d028c9193155af96aeb60 2024-12-02T21:14:46,747 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/87ba365a3e5d028c9193155af96aeb60 2024-12-02T21:14:46,748 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1048): stopping wal replay for 87ba365a3e5d028c9193155af96aeb60 2024-12-02T21:14:46,748 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1060): Cleaning up temporary data for 87ba365a3e5d028c9193155af96aeb60 2024-12-02T21:14:46,751 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1093): writing seq id for 87ba365a3e5d028c9193155af96aeb60 2024-12-02T21:14:46,753 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1114): Opened 87ba365a3e5d028c9193155af96aeb60; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62450272, jitterRate=-0.06941843032836914}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T21:14:46,753 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 87ba365a3e5d028c9193155af96aeb60 2024-12-02T21:14:46,753 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1006): Region open journal for 87ba365a3e5d028c9193155af96aeb60: Running coprocessor pre-open hook at 1733174086739Writing region info on filesystem at 1733174086739Initializing all the Stores at 1733174086740 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174086740Cleaning up temporary data from old regions at 1733174086748 (+8 ms)Running coprocessor post-open hooks at 1733174086753 (+5 ms)Region opened successfully at 1733174086753 2024-12-02T21:14:46,754 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2236): Post open deploy tasks for test,nnn,1733174081291.87ba365a3e5d028c9193155af96aeb60., pid=76, masterSystemTime=1733174086708 2024-12-02T21:14:46,754 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=true, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1845) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 4 more 2024-12-02T21:14:46,756 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-02T21:14:46,757 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=82, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1a4ea0dec946dcfe96ad162d5bcf5949, server=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:46,757 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T21:14:46,757 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2266): Finished post open deploy task for test,nnn,1733174081291.87ba365a3e5d028c9193155af96aeb60. 2024-12-02T21:14:46,757 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(153): Opened test,nnn,1733174081291.87ba365a3e5d028c9193155af96aeb60. 2024-12-02T21:14:46,757 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=82, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1a4ea0dec946dcfe96ad162d5bcf5949, server=87c3fdb6c570,40413,1733174078061 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:14:46,758 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(132): Open test,ggg,1733174081291.0dfd4169e4c254c51bcc70a472f99ecf. 2024-12-02T21:14:46,758 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7752): Opening region: {ENCODED => 0dfd4169e4c254c51bcc70a472f99ecf, NAME => 'test,ggg,1733174081291.0dfd4169e4c254c51bcc70a472f99ecf.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-12-02T21:14:46,758 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 0dfd4169e4c254c51bcc70a472f99ecf 2024-12-02T21:14:46,758 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(898): Instantiated test,ggg,1733174081291.0dfd4169e4c254c51bcc70a472f99ecf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:14:46,758 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7794): checking encryption for 0dfd4169e4c254c51bcc70a472f99ecf 2024-12-02T21:14:46,758 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7797): checking classloading for 0dfd4169e4c254c51bcc70a472f99ecf 2024-12-02T21:14:46,766 INFO [StoreOpener-0dfd4169e4c254c51bcc70a472f99ecf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 0dfd4169e4c254c51bcc70a472f99ecf 2024-12-02T21:14:46,768 INFO [StoreOpener-0dfd4169e4c254c51bcc70a472f99ecf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0dfd4169e4c254c51bcc70a472f99ecf columnFamilyName family 2024-12-02T21:14:46,768 DEBUG [StoreOpener-0dfd4169e4c254c51bcc70a472f99ecf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:14:46,769 INFO [StoreOpener-0dfd4169e4c254c51bcc70a472f99ecf-1 {}] regionserver.HStore(327): Store=0dfd4169e4c254c51bcc70a472f99ecf/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T21:14:46,769 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1038): replaying wal for 0dfd4169e4c254c51bcc70a472f99ecf 2024-12-02T21:14:46,770 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/0dfd4169e4c254c51bcc70a472f99ecf 2024-12-02T21:14:46,772 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/0dfd4169e4c254c51bcc70a472f99ecf 2024-12-02T21:14:46,772 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1048): stopping wal replay for 0dfd4169e4c254c51bcc70a472f99ecf 2024-12-02T21:14:46,772 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1060): Cleaning up temporary data for 0dfd4169e4c254c51bcc70a472f99ecf 2024-12-02T21:14:46,774 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): writing seq id for 0dfd4169e4c254c51bcc70a472f99ecf 2024-12-02T21:14:46,776 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1114): Opened 0dfd4169e4c254c51bcc70a472f99ecf; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64298528, jitterRate=-0.04187726974487305}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T21:14:46,776 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0dfd4169e4c254c51bcc70a472f99ecf 2024-12-02T21:14:46,776 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1006): Region open journal for 0dfd4169e4c254c51bcc70a472f99ecf: Running coprocessor pre-open hook at 1733174086759Writing region info on filesystem at 1733174086759Initializing all the Stores at 1733174086760 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174086760Cleaning up temporary data from old regions at 1733174086772 (+12 ms)Running coprocessor post-open hooks at 1733174086776 (+4 ms)Region opened successfully at 1733174086776 2024-12-02T21:14:46,777 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ggg,1733174081291.0dfd4169e4c254c51bcc70a472f99ecf., pid=75, masterSystemTime=1733174086708 2024-12-02T21:14:46,779 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ggg,1733174081291.0dfd4169e4c254c51bcc70a472f99ecf. 2024-12-02T21:14:46,779 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(153): Opened test,ggg,1733174081291.0dfd4169e4c254c51bcc70a472f99ecf. 2024-12-02T21:14:46,780 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open test,xxx,1733174081291.06d131e39cb4f5b02db4ec472d438a26. 2024-12-02T21:14:46,780 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => 06d131e39cb4f5b02db4ec472d438a26, NAME => 'test,xxx,1733174081291.06d131e39cb4f5b02db4ec472d438a26.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-12-02T21:14:46,780 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 06d131e39cb4f5b02db4ec472d438a26 2024-12-02T21:14:46,780 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated test,xxx,1733174081291.06d131e39cb4f5b02db4ec472d438a26.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:14:46,780 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for 06d131e39cb4f5b02db4ec472d438a26 2024-12-02T21:14:46,780 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for 06d131e39cb4f5b02db4ec472d438a26 2024-12-02T21:14:46,783 INFO [StoreOpener-06d131e39cb4f5b02db4ec472d438a26-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 06d131e39cb4f5b02db4ec472d438a26 2024-12-02T21:14:46,784 INFO [StoreOpener-06d131e39cb4f5b02db4ec472d438a26-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06d131e39cb4f5b02db4ec472d438a26 columnFamilyName family 2024-12-02T21:14:46,784 DEBUG [StoreOpener-06d131e39cb4f5b02db4ec472d438a26-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:14:46,785 INFO [StoreOpener-06d131e39cb4f5b02db4ec472d438a26-1 {}] regionserver.HStore(327): Store=06d131e39cb4f5b02db4ec472d438a26/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T21:14:46,785 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for 06d131e39cb4f5b02db4ec472d438a26 2024-12-02T21:14:46,786 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/06d131e39cb4f5b02db4ec472d438a26 2024-12-02T21:14:46,787 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/06d131e39cb4f5b02db4ec472d438a26 2024-12-02T21:14:46,787 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for 06d131e39cb4f5b02db4ec472d438a26 2024-12-02T21:14:46,788 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for 06d131e39cb4f5b02db4ec472d438a26 2024-12-02T21:14:46,789 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for 06d131e39cb4f5b02db4ec472d438a26 2024-12-02T21:14:46,790 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened 06d131e39cb4f5b02db4ec472d438a26; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63183331, jitterRate=-0.05849500000476837}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T21:14:46,790 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 06d131e39cb4f5b02db4ec472d438a26 2024-12-02T21:14:46,791 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for 06d131e39cb4f5b02db4ec472d438a26: Running coprocessor pre-open hook at 1733174086781Writing region info on filesystem at 1733174086781Initializing all the Stores at 1733174086782 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174086782Cleaning up temporary data from old regions at 1733174086788 (+6 ms)Running coprocessor post-open hooks at 1733174086790 (+2 ms)Region opened successfully at 1733174086791 (+1 ms) 2024-12-02T21:14:46,791 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1733174081291.06d131e39cb4f5b02db4ec472d438a26., pid=79, masterSystemTime=1733174086708 2024-12-02T21:14:46,794 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1733174081291.06d131e39cb4f5b02db4ec472d438a26. 2024-12-02T21:14:46,794 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(153): Opened test,xxx,1733174081291.06d131e39cb4f5b02db4ec472d438a26. 2024-12-02T21:14:46,814 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(132): Open test,lll,1733174081291.0d0be65f6ffd7ebb4c8b2fee37042136. 2024-12-02T21:14:46,814 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7752): Opening region: {ENCODED => 0d0be65f6ffd7ebb4c8b2fee37042136, NAME => 'test,lll,1733174081291.0d0be65f6ffd7ebb4c8b2fee37042136.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-12-02T21:14:46,815 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 0d0be65f6ffd7ebb4c8b2fee37042136 2024-12-02T21:14:46,815 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(898): Instantiated test,lll,1733174081291.0d0be65f6ffd7ebb4c8b2fee37042136.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T21:14:46,815 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7794): checking encryption for 0d0be65f6ffd7ebb4c8b2fee37042136 2024-12-02T21:14:46,815 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7797): checking classloading for 0d0be65f6ffd7ebb4c8b2fee37042136 2024-12-02T21:14:46,817 INFO [StoreOpener-0d0be65f6ffd7ebb4c8b2fee37042136-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 0d0be65f6ffd7ebb4c8b2fee37042136 2024-12-02T21:14:46,818 INFO [StoreOpener-0d0be65f6ffd7ebb4c8b2fee37042136-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d0be65f6ffd7ebb4c8b2fee37042136 columnFamilyName family 2024-12-02T21:14:46,818 DEBUG [StoreOpener-0d0be65f6ffd7ebb4c8b2fee37042136-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T21:14:46,818 INFO [StoreOpener-0d0be65f6ffd7ebb4c8b2fee37042136-1 {}] regionserver.HStore(327): Store=0d0be65f6ffd7ebb4c8b2fee37042136/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T21:14:46,819 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1038): replaying wal for 0d0be65f6ffd7ebb4c8b2fee37042136 2024-12-02T21:14:46,819 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/0d0be65f6ffd7ebb4c8b2fee37042136 2024-12-02T21:14:46,821 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/0d0be65f6ffd7ebb4c8b2fee37042136 2024-12-02T21:14:46,821 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1048): stopping wal replay for 0d0be65f6ffd7ebb4c8b2fee37042136 2024-12-02T21:14:46,821 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1060): Cleaning up temporary data for 0d0be65f6ffd7ebb4c8b2fee37042136 2024-12-02T21:14:46,823 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1093): writing seq id for 0d0be65f6ffd7ebb4c8b2fee37042136 2024-12-02T21:14:46,824 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-02T21:14:46,824 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T21:14:46,824 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-02T21:14:46,824 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1114): Opened 0d0be65f6ffd7ebb4c8b2fee37042136; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70541534, jitterRate=0.051150768995285034}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T21:14:46,824 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-02T21:14:46,825 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0d0be65f6ffd7ebb4c8b2fee37042136 2024-12-02T21:14:46,825 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1006): Region open journal for 0d0be65f6ffd7ebb4c8b2fee37042136: Running coprocessor pre-open hook at 1733174086815Writing region info on filesystem at 1733174086815Initializing all the Stores at 1733174086816 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733174086816Cleaning up temporary data from old regions at 1733174086821 (+5 ms)Running coprocessor post-open hooks at 1733174086825 (+4 ms)Region opened successfully at 1733174086825 2024-12-02T21:14:46,826 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-02T21:14:46,826 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-02T21:14:46,826 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-02T21:14:46,826 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1733174081291.0d0be65f6ffd7ebb4c8b2fee37042136., pid=77, masterSystemTime=1733174086810 2024-12-02T21:14:46,826 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-02T21:14:46,827 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-02T21:14:46,827 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42045 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:14:46,835 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-02T21:14:46,836 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T21:14:46,836 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T21:14:46,836 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:46,836 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T21:14:46,836 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:46,836 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=82, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T21:14:46,836 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:46,836 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T21:14:46,836 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:46,836 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T21:14:46,836 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:46,836 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=81, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T21:14:46,837 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:46,837 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T21:14:46,837 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:46,837 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T21:14:46,837 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:46,837 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T21:14:46,837 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T21:14:46,837 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T21:14:46,837 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:46,837 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T21:14:46,838 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-02T21:14:46,839 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-02T21:14:46,840 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=120cf3776e92a15d3276f11911c4042f, REOPEN/MOVE 2024-12-02T21:14:46,845 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true 2024-12-02T21:14:46,845 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:46,845 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:46,845 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T21:14:46,845 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:46,845 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:46,845 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:46,845 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN 2024-12-02T21:14:46,846 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN 2024-12-02T21:14:46,846 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN 2024-12-02T21:14:46,846 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=87ba365a3e5d028c9193155af96aeb60, ASSIGN 2024-12-02T21:14:46,846 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b, ASSIGN 2024-12-02T21:14:46,846 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0dfd4169e4c254c51bcc70a472f99ecf, ASSIGN 2024-12-02T21:14:46,846 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d63da23e8236e18b416716ec838dd43e, ASSIGN 2024-12-02T21:14:46,846 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d0be65f6ffd7ebb4c8b2fee37042136, ASSIGN 2024-12-02T21:14:46,846 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0dfd4169e4c254c51bcc70a472f99ecf, server=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:46,846 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 87ba365a3e5d028c9193155af96aeb60, server=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:46,846 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0d0be65f6ffd7ebb4c8b2fee37042136, server=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:46,846 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=73, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:46,846 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06d131e39cb4f5b02db4ec472d438a26, server=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:46,846 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0517912ff18e81317bb5474cefaca025, server=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:46,846 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=81, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure a50bae1d5cb3d03a86dc103b09a0f68b, server=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:46,846 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=82, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1a4ea0dec946dcfe96ad162d5bcf5949, server=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:46,846 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:46,846 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:46,847 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:46,847 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:46,847 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:46,847 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:46,847 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:46,847 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:46,847 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN 2024-12-02T21:14:46,847 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:46,847 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN 2024-12-02T21:14:46,847 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:46,847 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN 2024-12-02T21:14:46,847 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=87ba365a3e5d028c9193155af96aeb60, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:46,847 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=87ba365a3e5d028c9193155af96aeb60, ASSIGN 2024-12-02T21:14:46,847 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:46,847 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b, ASSIGN 2024-12-02T21:14:46,847 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0dfd4169e4c254c51bcc70a472f99ecf, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:46,847 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0dfd4169e4c254c51bcc70a472f99ecf, ASSIGN 2024-12-02T21:14:46,847 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d63da23e8236e18b416716ec838dd43e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:46,847 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d63da23e8236e18b416716ec838dd43e, ASSIGN 2024-12-02T21:14:46,848 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d0be65f6ffd7ebb4c8b2fee37042136, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:46,848 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d0be65f6ffd7ebb4c8b2fee37042136, ASSIGN 2024-12-02T21:14:46,848 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0dfd4169e4c254c51bcc70a472f99ecf, server=87c3fdb6c570,36585,1733174078182 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:46,848 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 87ba365a3e5d028c9193155af96aeb60, server=87c3fdb6c570,36585,1733174078182 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:46,848 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0d0be65f6ffd7ebb4c8b2fee37042136, server=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:46,848 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=73, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:46,848 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06d131e39cb4f5b02db4ec472d438a26, server=87c3fdb6c570,36585,1733174078182 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:46,848 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0517912ff18e81317bb5474cefaca025, server=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:46,848 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=81, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure a50bae1d5cb3d03a86dc103b09a0f68b, server=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:46,848 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=82, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1a4ea0dec946dcfe96ad162d5bcf5949, server=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:46,849 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 21 msec 2024-12-02T21:14:46,851 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=65 2024-12-02T21:14:46,851 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=87c3fdb6c570,40413,1733174078061, table=hbase:meta, region=1588230740 2024-12-02T21:14:46,851 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 87c3fdb6c570,40413,1733174078061, state=OPEN 2024-12-02T21:14:46,854 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "87c3fdb6c570" port: 40413 start_code: 1733174078061 } transition { transition_code: OPENED region_info { region_id: 1733174081291 table_name { namespace: "default" qualifier: "test" } start_key: "lll" end_key: "mmm" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 77 initiating_master_active_time: 1733174078474 }; retry (#0) after 1002ms delay (Master is coming online...). org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-12-02T21:14:46,862 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40413-0x10197e5a0060001, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:46,862 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36585-0x10197e5a0060002, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:46,862 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42045-0x10197e5a0060000, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:46,862 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:46,862 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:46,862 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:46,863 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN to state=OFFLINE, location=null, table=test, region=06d131e39cb4f5b02db4ec472d438a26 to restore RIT 2024-12-02T21:14:46,864 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN to state=OFFLINE, location=null, table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949 to restore RIT 2024-12-02T21:14:46,864 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN to state=OFFLINE, location=null, table=test, region=0517912ff18e81317bb5474cefaca025 to restore RIT 2024-12-02T21:14:46,864 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=87ba365a3e5d028c9193155af96aeb60, ASSIGN to state=OFFLINE, location=null, table=test, region=87ba365a3e5d028c9193155af96aeb60 to restore RIT 2024-12-02T21:14:46,864 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b, ASSIGN to state=OFFLINE, location=null, table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b to restore RIT 2024-12-02T21:14:46,864 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0dfd4169e4c254c51bcc70a472f99ecf, ASSIGN to state=OFFLINE, location=null, table=test, region=0dfd4169e4c254c51bcc70a472f99ecf to restore RIT 2024-12-02T21:14:46,864 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d63da23e8236e18b416716ec838dd43e, ASSIGN to state=OFFLINE, location=null, table=test, region=d63da23e8236e18b416716ec838dd43e to restore RIT 2024-12-02T21:14:46,864 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d0be65f6ffd7ebb4c8b2fee37042136, ASSIGN to state=OFFLINE, location=null, table=test, region=0d0be65f6ffd7ebb4c8b2fee37042136 to restore RIT 2024-12-02T21:14:46,865 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733174116865 2024-12-02T21:14:46,865 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-02T21:14:46,865 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:14:46,867 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-02T21:14:46,868 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-02T21:14:46,868 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=81, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure a50bae1d5cb3d03a86dc103b09a0f68b, server=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:46,868 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T21:14:46,868 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=81, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure a50bae1d5cb3d03a86dc103b09a0f68b, server=87c3fdb6c570,40413,1733174078061 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:14:46,893 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85ca3a9a09d75c0941c58f5b83584a8e, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:46,894 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ed86f04f519cc5dccd1b6d41be3da7d, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:46,894 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0517912ff18e81317bb5474cefaca025, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:46,894 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff611163b4ab91c50d940651a0065c34, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:46,894 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=120cf3776e92a15d3276f11911c4042f, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=5 2024-12-02T21:14:46,895 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e87d99b937228303e28b7ea0373ddc70, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:46,895 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0dfd4169e4c254c51bcc70a472f99ecf, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:46,895 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9950441eda7548fa66872811a13f5bf0, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:46,895 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e039115fcc0d0a54e2d03efed7e05de, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:46,896 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d60c566b39516dd875aaf2bd7def402, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:46,896 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a524d88be9ac29b416f0a6c04627fa1c, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:46,896 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0d0be65f6ffd7ebb4c8b2fee37042136, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:46,896 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=292fd0b4e0f07516cc25c069181e3111, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:46,897 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87ba365a3e5d028c9193155af96aeb60, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:46,897 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4197cd4f7ea2f3d6256be6836e9834da, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:46,897 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=20545c8ee2d66091a91ff9697d1f18ca, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:46,897 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a50bae1d5cb3d03a86dc103b09a0f68b, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:46,898 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=215753e705337ae11b1b38ad77b6a5cd, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:46,898 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e42e4828e881f12c74fbedcdb6b00ea2, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:46,898 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d63da23e8236e18b416716ec838dd43e, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:46,898 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e40e25a9323e1d3dad4ec92c578e2f82, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:46,899 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a2fb6e9d39bf04774472d3b4c70810a, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:46,899 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1a4ea0dec946dcfe96ad162d5bcf5949, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:46,899 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06d131e39cb4f5b02db4ec472d438a26, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:46,899 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0875ddf9cbd035a3345be0684b07b2e6, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:46,900 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a36d8f99c2aef77f88827585926ad989, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:46,900 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-02T21:14:46,900 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733174146900 2024-12-02T21:14:46,900 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733174206900 2024-12-02T21:14:46,900 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 34 msec 2024-12-02T21:14:46,900 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-02T21:14:46,900 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T21:14:46,900 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-02T21:14:46,900 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-02T21:14:46,901 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-02T21:14:46,901 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-02T21:14:46,901 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-02T21:14:46,901 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-02T21:14:46,902 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-02T21:14:46,907 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-02T21:14:46,908 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T21:14:46,908 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T21:14:46,908 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:46,908 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T21:14:46,908 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:46,908 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:46,908 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T21:14:46,908 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:46,908 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T21:14:46,908 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:46,908 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=81, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T21:14:46,908 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:46,908 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T21:14:46,908 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:46,908 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T21:14:46,908 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:46,909 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T21:14:46,909 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T21:14:46,909 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T21:14:46,909 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:46,909 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T21:14:46,909 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-02T21:14:46,910 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-02T21:14:46,911 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=120cf3776e92a15d3276f11911c4042f, REOPEN/MOVE 2024-12-02T21:14:46,914 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true 2024-12-02T21:14:46,914 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:46,914 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:46,914 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T21:14:46,914 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:46,914 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:46,914 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:46,914 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN 2024-12-02T21:14:46,914 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN 2024-12-02T21:14:46,914 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN 2024-12-02T21:14:46,915 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=87ba365a3e5d028c9193155af96aeb60, ASSIGN 2024-12-02T21:14:46,915 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b, ASSIGN 2024-12-02T21:14:46,915 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0dfd4169e4c254c51bcc70a472f99ecf, ASSIGN 2024-12-02T21:14:46,915 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d63da23e8236e18b416716ec838dd43e, ASSIGN 2024-12-02T21:14:46,915 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d0be65f6ffd7ebb4c8b2fee37042136, ASSIGN 2024-12-02T21:14:46,915 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0dfd4169e4c254c51bcc70a472f99ecf, server=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:46,915 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 87ba365a3e5d028c9193155af96aeb60, server=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:46,915 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0d0be65f6ffd7ebb4c8b2fee37042136, server=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:46,915 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=73, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:46,915 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06d131e39cb4f5b02db4ec472d438a26, server=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:46,915 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0517912ff18e81317bb5474cefaca025, server=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:46,915 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=81, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure a50bae1d5cb3d03a86dc103b09a0f68b, server=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:46,915 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:46,915 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:46,915 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:46,915 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:46,915 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:46,915 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:46,915 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:46,915 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:46,915 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN 2024-12-02T21:14:46,915 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:46,915 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN 2024-12-02T21:14:46,915 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:46,915 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN 2024-12-02T21:14:46,915 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=87ba365a3e5d028c9193155af96aeb60, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:46,915 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=87ba365a3e5d028c9193155af96aeb60, ASSIGN 2024-12-02T21:14:46,915 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:46,915 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b, ASSIGN 2024-12-02T21:14:46,915 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0dfd4169e4c254c51bcc70a472f99ecf, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:46,915 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0dfd4169e4c254c51bcc70a472f99ecf, ASSIGN 2024-12-02T21:14:46,916 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d63da23e8236e18b416716ec838dd43e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:46,916 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d63da23e8236e18b416716ec838dd43e, ASSIGN 2024-12-02T21:14:46,916 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d0be65f6ffd7ebb4c8b2fee37042136, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:46,916 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d0be65f6ffd7ebb4c8b2fee37042136, ASSIGN 2024-12-02T21:14:46,916 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0dfd4169e4c254c51bcc70a472f99ecf, server=87c3fdb6c570,36585,1733174078182 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:46,916 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 87ba365a3e5d028c9193155af96aeb60, server=87c3fdb6c570,36585,1733174078182 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:46,916 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0d0be65f6ffd7ebb4c8b2fee37042136, server=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:46,916 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=73, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:46,916 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06d131e39cb4f5b02db4ec472d438a26, server=87c3fdb6c570,36585,1733174078182 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:46,916 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0517912ff18e81317bb5474cefaca025, server=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:46,916 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=81, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure a50bae1d5cb3d03a86dc103b09a0f68b, server=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:46,916 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 13 msec 2024-12-02T21:14:46,917 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=65 2024-12-02T21:14:46,917 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=87c3fdb6c570,40413,1733174078061, table=hbase:meta, region=1588230740 2024-12-02T21:14:46,917 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 87c3fdb6c570,40413,1733174078061, state=OPEN 2024-12-02T21:14:46,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42045-0x10197e5a0060000, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:46,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36585-0x10197e5a0060002, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:46,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40413-0x10197e5a0060001, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:46,929 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:46,929 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:46,929 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:46,930 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN to state=OFFLINE, location=null, table=test, region=06d131e39cb4f5b02db4ec472d438a26 to restore RIT 2024-12-02T21:14:46,930 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN to state=OFFLINE, location=null, table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949 to restore RIT 2024-12-02T21:14:46,930 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN to state=OFFLINE, location=null, table=test, region=0517912ff18e81317bb5474cefaca025 to restore RIT 2024-12-02T21:14:46,930 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=87ba365a3e5d028c9193155af96aeb60, ASSIGN to state=OFFLINE, location=null, table=test, region=87ba365a3e5d028c9193155af96aeb60 to restore RIT 2024-12-02T21:14:46,930 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b, ASSIGN to state=OFFLINE, location=null, table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b to restore RIT 2024-12-02T21:14:46,930 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0dfd4169e4c254c51bcc70a472f99ecf, ASSIGN to state=OFFLINE, location=null, table=test, region=0dfd4169e4c254c51bcc70a472f99ecf to restore RIT 2024-12-02T21:14:46,930 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d63da23e8236e18b416716ec838dd43e, ASSIGN to state=OFFLINE, location=null, table=test, region=d63da23e8236e18b416716ec838dd43e to restore RIT 2024-12-02T21:14:46,930 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d0be65f6ffd7ebb4c8b2fee37042136, ASSIGN to state=OFFLINE, location=null, table=test, region=0d0be65f6ffd7ebb4c8b2fee37042136 to restore RIT 2024-12-02T21:14:46,931 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733174116931 2024-12-02T21:14:46,931 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-02T21:14:46,931 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:14:46,932 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-02T21:14:46,933 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-02T21:14:46,933 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=80, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0517912ff18e81317bb5474cefaca025, server=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:46,933 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T21:14:46,933 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=80, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0517912ff18e81317bb5474cefaca025, server=87c3fdb6c570,40413,1733174078061 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:14:46,941 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85ca3a9a09d75c0941c58f5b83584a8e, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:46,941 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ed86f04f519cc5dccd1b6d41be3da7d, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:46,941 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0517912ff18e81317bb5474cefaca025, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:46,941 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff611163b4ab91c50d940651a0065c34, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:46,942 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=120cf3776e92a15d3276f11911c4042f, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=5 2024-12-02T21:14:46,942 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e87d99b937228303e28b7ea0373ddc70, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:46,942 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0dfd4169e4c254c51bcc70a472f99ecf, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:46,942 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9950441eda7548fa66872811a13f5bf0, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:46,943 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e039115fcc0d0a54e2d03efed7e05de, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:46,943 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d60c566b39516dd875aaf2bd7def402, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:46,943 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a524d88be9ac29b416f0a6c04627fa1c, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:46,943 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0d0be65f6ffd7ebb4c8b2fee37042136, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:46,943 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=292fd0b4e0f07516cc25c069181e3111, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:46,944 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87ba365a3e5d028c9193155af96aeb60, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:46,944 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4197cd4f7ea2f3d6256be6836e9834da, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:46,944 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=20545c8ee2d66091a91ff9697d1f18ca, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:46,944 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a50bae1d5cb3d03a86dc103b09a0f68b, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:46,945 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=215753e705337ae11b1b38ad77b6a5cd, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:46,945 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e42e4828e881f12c74fbedcdb6b00ea2, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:46,945 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d63da23e8236e18b416716ec838dd43e, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:46,945 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e40e25a9323e1d3dad4ec92c578e2f82, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:46,945 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a2fb6e9d39bf04774472d3b4c70810a, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:46,946 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1a4ea0dec946dcfe96ad162d5bcf5949, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:46,946 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06d131e39cb4f5b02db4ec472d438a26, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:46,946 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0875ddf9cbd035a3345be0684b07b2e6, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:46,946 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a36d8f99c2aef77f88827585926ad989, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:46,946 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-02T21:14:46,946 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733174146946 2024-12-02T21:14:46,947 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733174206947 2024-12-02T21:14:46,947 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 16 msec 2024-12-02T21:14:46,947 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-02T21:14:46,947 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T21:14:46,947 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-02T21:14:46,947 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-02T21:14:46,947 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-02T21:14:46,947 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-02T21:14:46,947 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-02T21:14:46,948 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-02T21:14:46,949 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-02T21:14:46,954 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-02T21:14:46,954 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T21:14:46,954 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T21:14:46,954 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:46,954 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T21:14:46,954 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:46,954 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:46,954 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T21:14:46,954 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:46,954 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T21:14:46,954 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:46,954 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:46,955 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T21:14:46,955 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:46,955 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T21:14:46,955 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:46,955 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T21:14:46,955 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T21:14:46,955 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T21:14:46,955 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:46,955 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T21:14:46,955 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-02T21:14:46,956 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-02T21:14:46,957 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=120cf3776e92a15d3276f11911c4042f, REOPEN/MOVE 2024-12-02T21:14:46,959 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true 2024-12-02T21:14:46,959 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:46,960 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:46,960 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T21:14:46,960 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:46,960 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:46,960 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:46,960 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN 2024-12-02T21:14:46,960 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN 2024-12-02T21:14:46,960 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN 2024-12-02T21:14:46,960 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=87ba365a3e5d028c9193155af96aeb60, ASSIGN 2024-12-02T21:14:46,960 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b, ASSIGN 2024-12-02T21:14:46,960 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0dfd4169e4c254c51bcc70a472f99ecf, ASSIGN 2024-12-02T21:14:46,960 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d63da23e8236e18b416716ec838dd43e, ASSIGN 2024-12-02T21:14:46,960 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d0be65f6ffd7ebb4c8b2fee37042136, ASSIGN 2024-12-02T21:14:46,960 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0dfd4169e4c254c51bcc70a472f99ecf, server=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:46,960 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 87ba365a3e5d028c9193155af96aeb60, server=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:46,960 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0d0be65f6ffd7ebb4c8b2fee37042136, server=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:46,960 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=73, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:46,960 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06d131e39cb4f5b02db4ec472d438a26, server=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:46,960 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0517912ff18e81317bb5474cefaca025, server=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:46,960 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:46,961 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:46,961 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:46,961 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:46,961 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:46,961 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:46,961 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:46,961 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:46,961 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN 2024-12-02T21:14:46,961 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:46,961 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN 2024-12-02T21:14:46,961 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:46,961 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN 2024-12-02T21:14:46,961 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=87ba365a3e5d028c9193155af96aeb60, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:46,961 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=87ba365a3e5d028c9193155af96aeb60, ASSIGN 2024-12-02T21:14:46,961 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:46,961 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b, ASSIGN 2024-12-02T21:14:46,961 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0dfd4169e4c254c51bcc70a472f99ecf, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:46,961 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0dfd4169e4c254c51bcc70a472f99ecf, ASSIGN 2024-12-02T21:14:46,961 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d63da23e8236e18b416716ec838dd43e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:46,961 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d63da23e8236e18b416716ec838dd43e, ASSIGN 2024-12-02T21:14:46,961 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d0be65f6ffd7ebb4c8b2fee37042136, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:46,962 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d0be65f6ffd7ebb4c8b2fee37042136, ASSIGN 2024-12-02T21:14:46,962 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0dfd4169e4c254c51bcc70a472f99ecf, server=87c3fdb6c570,36585,1733174078182 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:46,962 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 87ba365a3e5d028c9193155af96aeb60, server=87c3fdb6c570,36585,1733174078182 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:46,962 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0d0be65f6ffd7ebb4c8b2fee37042136, server=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:46,962 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=73, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:46,962 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06d131e39cb4f5b02db4ec472d438a26, server=87c3fdb6c570,36585,1733174078182 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:46,962 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0517912ff18e81317bb5474cefaca025, server=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:46,962 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 13 msec 2024-12-02T21:14:46,963 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=65 2024-12-02T21:14:46,963 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=87c3fdb6c570,40413,1733174078061, table=hbase:meta, region=1588230740 2024-12-02T21:14:46,964 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 87c3fdb6c570,40413,1733174078061, state=OPEN 2024-12-02T21:14:46,971 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36585-0x10197e5a0060002, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:46,971 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40413-0x10197e5a0060001, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:46,971 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42045-0x10197e5a0060000, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:46,971 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:46,971 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:46,971 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:46,972 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN to state=OFFLINE, location=null, table=test, region=06d131e39cb4f5b02db4ec472d438a26 to restore RIT 2024-12-02T21:14:46,972 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN to state=OFFLINE, location=null, table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949 to restore RIT 2024-12-02T21:14:46,972 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN to state=OFFLINE, location=null, table=test, region=0517912ff18e81317bb5474cefaca025 to restore RIT 2024-12-02T21:14:46,972 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=87ba365a3e5d028c9193155af96aeb60, ASSIGN to state=OFFLINE, location=null, table=test, region=87ba365a3e5d028c9193155af96aeb60 to restore RIT 2024-12-02T21:14:46,972 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b, ASSIGN to state=OFFLINE, location=null, table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b to restore RIT 2024-12-02T21:14:46,972 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0dfd4169e4c254c51bcc70a472f99ecf, ASSIGN to state=OFFLINE, location=null, table=test, region=0dfd4169e4c254c51bcc70a472f99ecf to restore RIT 2024-12-02T21:14:46,972 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d63da23e8236e18b416716ec838dd43e, ASSIGN to state=OFFLINE, location=null, table=test, region=d63da23e8236e18b416716ec838dd43e to restore RIT 2024-12-02T21:14:46,972 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d0be65f6ffd7ebb4c8b2fee37042136, ASSIGN to state=OFFLINE, location=null, table=test, region=0d0be65f6ffd7ebb4c8b2fee37042136 to restore RIT 2024-12-02T21:14:46,973 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733174116973 2024-12-02T21:14:46,973 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-02T21:14:46,973 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:14:46,974 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-02T21:14:46,975 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-02T21:14:46,975 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=79, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06d131e39cb4f5b02db4ec472d438a26, server=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:46,975 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T21:14:46,975 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=79, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06d131e39cb4f5b02db4ec472d438a26, server=87c3fdb6c570,36585,1733174078182 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:14:46,981 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85ca3a9a09d75c0941c58f5b83584a8e, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:46,982 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ed86f04f519cc5dccd1b6d41be3da7d, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:46,982 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0517912ff18e81317bb5474cefaca025, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:46,982 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff611163b4ab91c50d940651a0065c34, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:46,982 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=120cf3776e92a15d3276f11911c4042f, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=5 2024-12-02T21:14:46,983 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e87d99b937228303e28b7ea0373ddc70, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:46,983 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0dfd4169e4c254c51bcc70a472f99ecf, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:46,983 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9950441eda7548fa66872811a13f5bf0, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:46,983 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e039115fcc0d0a54e2d03efed7e05de, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:46,984 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d60c566b39516dd875aaf2bd7def402, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:46,984 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a524d88be9ac29b416f0a6c04627fa1c, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:46,984 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0d0be65f6ffd7ebb4c8b2fee37042136, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:46,985 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=292fd0b4e0f07516cc25c069181e3111, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:46,985 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87ba365a3e5d028c9193155af96aeb60, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:46,985 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4197cd4f7ea2f3d6256be6836e9834da, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:46,986 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=20545c8ee2d66091a91ff9697d1f18ca, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:46,986 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a50bae1d5cb3d03a86dc103b09a0f68b, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:46,986 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=215753e705337ae11b1b38ad77b6a5cd, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:46,986 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e42e4828e881f12c74fbedcdb6b00ea2, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:46,986 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d63da23e8236e18b416716ec838dd43e, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:46,987 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e40e25a9323e1d3dad4ec92c578e2f82, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:46,987 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a2fb6e9d39bf04774472d3b4c70810a, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:46,987 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1a4ea0dec946dcfe96ad162d5bcf5949, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:46,987 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06d131e39cb4f5b02db4ec472d438a26, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:46,987 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0875ddf9cbd035a3345be0684b07b2e6, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:46,988 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a36d8f99c2aef77f88827585926ad989, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:46,988 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-02T21:14:46,988 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733174146988 2024-12-02T21:14:46,988 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733174206988 2024-12-02T21:14:46,988 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 15 msec 2024-12-02T21:14:46,988 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-02T21:14:46,988 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T21:14:46,988 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-02T21:14:46,988 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-02T21:14:46,989 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-02T21:14:46,989 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-02T21:14:46,989 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-02T21:14:46,990 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-02T21:14:46,990 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-02T21:14:46,995 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-02T21:14:46,995 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T21:14:46,995 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T21:14:46,995 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:46,995 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T21:14:46,995 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:46,995 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:46,995 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:46,996 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T21:14:46,996 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:46,996 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:46,996 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T21:14:46,996 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:46,996 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T21:14:46,996 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:46,996 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T21:14:46,996 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T21:14:46,996 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T21:14:46,996 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:46,996 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T21:14:46,997 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-02T21:14:46,998 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-02T21:14:46,998 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=120cf3776e92a15d3276f11911c4042f, REOPEN/MOVE 2024-12-02T21:14:47,002 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true 2024-12-02T21:14:47,002 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:47,002 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:47,002 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T21:14:47,002 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:47,002 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:47,002 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:47,002 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN 2024-12-02T21:14:47,002 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN 2024-12-02T21:14:47,002 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN 2024-12-02T21:14:47,002 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=87ba365a3e5d028c9193155af96aeb60, ASSIGN 2024-12-02T21:14:47,003 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b, ASSIGN 2024-12-02T21:14:47,003 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0dfd4169e4c254c51bcc70a472f99ecf, ASSIGN 2024-12-02T21:14:47,003 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d63da23e8236e18b416716ec838dd43e, ASSIGN 2024-12-02T21:14:47,003 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d0be65f6ffd7ebb4c8b2fee37042136, ASSIGN 2024-12-02T21:14:47,003 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0dfd4169e4c254c51bcc70a472f99ecf, server=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:47,003 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 87ba365a3e5d028c9193155af96aeb60, server=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:47,003 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0d0be65f6ffd7ebb4c8b2fee37042136, server=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:47,003 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=73, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:47,003 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06d131e39cb4f5b02db4ec472d438a26, server=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:47,003 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,003 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,003 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,003 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,003 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,003 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,003 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,003 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,003 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN 2024-12-02T21:14:47,003 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,003 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN 2024-12-02T21:14:47,003 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,003 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN 2024-12-02T21:14:47,003 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=87ba365a3e5d028c9193155af96aeb60, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,003 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=87ba365a3e5d028c9193155af96aeb60, ASSIGN 2024-12-02T21:14:47,003 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,004 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b, ASSIGN 2024-12-02T21:14:47,004 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0dfd4169e4c254c51bcc70a472f99ecf, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,004 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0dfd4169e4c254c51bcc70a472f99ecf, ASSIGN 2024-12-02T21:14:47,004 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d63da23e8236e18b416716ec838dd43e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,004 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d63da23e8236e18b416716ec838dd43e, ASSIGN 2024-12-02T21:14:47,004 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d0be65f6ffd7ebb4c8b2fee37042136, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,004 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d0be65f6ffd7ebb4c8b2fee37042136, ASSIGN 2024-12-02T21:14:47,004 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0dfd4169e4c254c51bcc70a472f99ecf, server=87c3fdb6c570,36585,1733174078182 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,004 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 87ba365a3e5d028c9193155af96aeb60, server=87c3fdb6c570,36585,1733174078182 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,004 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0d0be65f6ffd7ebb4c8b2fee37042136, server=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,004 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=73, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,004 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06d131e39cb4f5b02db4ec472d438a26, server=87c3fdb6c570,36585,1733174078182 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,004 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 13 msec 2024-12-02T21:14:47,005 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=65 2024-12-02T21:14:47,005 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=87c3fdb6c570,40413,1733174078061, table=hbase:meta, region=1588230740 2024-12-02T21:14:47,005 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 87c3fdb6c570,40413,1733174078061, state=OPEN 2024-12-02T21:14:47,013 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42045-0x10197e5a0060000, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:47,013 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36585-0x10197e5a0060002, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:47,013 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40413-0x10197e5a0060001, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:47,013 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:47,013 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:47,013 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:47,014 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN to state=OFFLINE, location=null, table=test, region=06d131e39cb4f5b02db4ec472d438a26 to restore RIT 2024-12-02T21:14:47,014 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN to state=OFFLINE, location=null, table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949 to restore RIT 2024-12-02T21:14:47,014 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN to state=OFFLINE, location=null, table=test, region=0517912ff18e81317bb5474cefaca025 to restore RIT 2024-12-02T21:14:47,014 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=87ba365a3e5d028c9193155af96aeb60, ASSIGN to state=OFFLINE, location=null, table=test, region=87ba365a3e5d028c9193155af96aeb60 to restore RIT 2024-12-02T21:14:47,014 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b, ASSIGN to state=OFFLINE, location=null, table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b to restore RIT 2024-12-02T21:14:47,014 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0dfd4169e4c254c51bcc70a472f99ecf, ASSIGN to state=OFFLINE, location=null, table=test, region=0dfd4169e4c254c51bcc70a472f99ecf to restore RIT 2024-12-02T21:14:47,014 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d63da23e8236e18b416716ec838dd43e, ASSIGN to state=OFFLINE, location=null, table=test, region=d63da23e8236e18b416716ec838dd43e to restore RIT 2024-12-02T21:14:47,014 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d0be65f6ffd7ebb4c8b2fee37042136, ASSIGN to state=OFFLINE, location=null, table=test, region=0d0be65f6ffd7ebb4c8b2fee37042136 to restore RIT 2024-12-02T21:14:47,015 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733174117014 2024-12-02T21:14:47,015 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-02T21:14:47,015 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:14:47,016 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-02T21:14:47,017 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-02T21:14:47,017 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=78, ppid=73, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:47,017 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T21:14:47,017 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=78, ppid=73, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:14:47,022 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85ca3a9a09d75c0941c58f5b83584a8e, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,022 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ed86f04f519cc5dccd1b6d41be3da7d, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,022 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0517912ff18e81317bb5474cefaca025, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,022 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff611163b4ab91c50d940651a0065c34, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,023 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=120cf3776e92a15d3276f11911c4042f, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=5 2024-12-02T21:14:47,023 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e87d99b937228303e28b7ea0373ddc70, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,023 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0dfd4169e4c254c51bcc70a472f99ecf, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,023 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9950441eda7548fa66872811a13f5bf0, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,023 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e039115fcc0d0a54e2d03efed7e05de, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,023 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d60c566b39516dd875aaf2bd7def402, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,023 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a524d88be9ac29b416f0a6c04627fa1c, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,023 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0d0be65f6ffd7ebb4c8b2fee37042136, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,024 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=292fd0b4e0f07516cc25c069181e3111, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,024 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87ba365a3e5d028c9193155af96aeb60, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,024 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4197cd4f7ea2f3d6256be6836e9834da, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,024 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=20545c8ee2d66091a91ff9697d1f18ca, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,024 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a50bae1d5cb3d03a86dc103b09a0f68b, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,024 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=215753e705337ae11b1b38ad77b6a5cd, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,024 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e42e4828e881f12c74fbedcdb6b00ea2, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,024 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d63da23e8236e18b416716ec838dd43e, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,024 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e40e25a9323e1d3dad4ec92c578e2f82, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,025 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a2fb6e9d39bf04774472d3b4c70810a, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,025 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1a4ea0dec946dcfe96ad162d5bcf5949, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,025 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06d131e39cb4f5b02db4ec472d438a26, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,025 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0875ddf9cbd035a3345be0684b07b2e6, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,025 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a36d8f99c2aef77f88827585926ad989, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,025 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-02T21:14:47,025 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733174147025 2024-12-02T21:14:47,025 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733174207025 2024-12-02T21:14:47,025 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-12-02T21:14:47,025 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-02T21:14:47,025 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T21:14:47,025 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-02T21:14:47,025 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-02T21:14:47,026 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-02T21:14:47,026 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-02T21:14:47,026 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-02T21:14:47,026 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-02T21:14:47,027 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-02T21:14:47,029 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-02T21:14:47,029 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T21:14:47,029 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T21:14:47,029 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,029 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,029 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,029 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,029 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T21:14:47,029 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,029 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,029 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T21:14:47,029 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,029 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T21:14:47,029 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,030 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T21:14:47,030 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T21:14:47,030 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T21:14:47,030 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,030 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T21:14:47,030 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-02T21:14:47,031 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-02T21:14:47,031 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=120cf3776e92a15d3276f11911c4042f, REOPEN/MOVE 2024-12-02T21:14:47,032 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true 2024-12-02T21:14:47,032 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:47,032 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:47,033 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T21:14:47,033 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:47,033 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:47,033 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:47,033 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN 2024-12-02T21:14:47,033 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN 2024-12-02T21:14:47,033 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN 2024-12-02T21:14:47,033 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=87ba365a3e5d028c9193155af96aeb60, ASSIGN 2024-12-02T21:14:47,033 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b, ASSIGN 2024-12-02T21:14:47,033 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0dfd4169e4c254c51bcc70a472f99ecf, ASSIGN 2024-12-02T21:14:47,033 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d63da23e8236e18b416716ec838dd43e, ASSIGN 2024-12-02T21:14:47,033 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d0be65f6ffd7ebb4c8b2fee37042136, ASSIGN 2024-12-02T21:14:47,033 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0dfd4169e4c254c51bcc70a472f99ecf, server=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:47,033 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 87ba365a3e5d028c9193155af96aeb60, server=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:47,033 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0d0be65f6ffd7ebb4c8b2fee37042136, server=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:47,033 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=73, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:47,033 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,033 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,033 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,033 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,033 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,033 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,033 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,033 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,033 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN 2024-12-02T21:14:47,033 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,033 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN 2024-12-02T21:14:47,033 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,033 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN 2024-12-02T21:14:47,034 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=87ba365a3e5d028c9193155af96aeb60, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,034 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=87ba365a3e5d028c9193155af96aeb60, ASSIGN 2024-12-02T21:14:47,034 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,034 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b, ASSIGN 2024-12-02T21:14:47,034 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0dfd4169e4c254c51bcc70a472f99ecf, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,034 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0dfd4169e4c254c51bcc70a472f99ecf, ASSIGN 2024-12-02T21:14:47,034 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d63da23e8236e18b416716ec838dd43e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,034 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d63da23e8236e18b416716ec838dd43e, ASSIGN 2024-12-02T21:14:47,034 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d0be65f6ffd7ebb4c8b2fee37042136, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,034 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d0be65f6ffd7ebb4c8b2fee37042136, ASSIGN 2024-12-02T21:14:47,034 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0dfd4169e4c254c51bcc70a472f99ecf, server=87c3fdb6c570,36585,1733174078182 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,034 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 87ba365a3e5d028c9193155af96aeb60, server=87c3fdb6c570,36585,1733174078182 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,034 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0d0be65f6ffd7ebb4c8b2fee37042136, server=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,034 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=73, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,034 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-12-02T21:14:47,037 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=65 2024-12-02T21:14:47,037 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=87c3fdb6c570,40413,1733174078061, table=hbase:meta, region=1588230740 2024-12-02T21:14:47,037 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 87c3fdb6c570,40413,1733174078061, state=OPEN 2024-12-02T21:14:47,045 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36585-0x10197e5a0060002, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:47,045 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42045-0x10197e5a0060000, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:47,045 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40413-0x10197e5a0060001, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:47,045 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:47,045 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:47,045 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:47,045 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN to state=OFFLINE, location=null, table=test, region=06d131e39cb4f5b02db4ec472d438a26 to restore RIT 2024-12-02T21:14:47,046 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN to state=OFFLINE, location=null, table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949 to restore RIT 2024-12-02T21:14:47,046 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN to state=OFFLINE, location=null, table=test, region=0517912ff18e81317bb5474cefaca025 to restore RIT 2024-12-02T21:14:47,046 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=87ba365a3e5d028c9193155af96aeb60, ASSIGN to state=OFFLINE, location=null, table=test, region=87ba365a3e5d028c9193155af96aeb60 to restore RIT 2024-12-02T21:14:47,046 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b, ASSIGN to state=OFFLINE, location=null, table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b to restore RIT 2024-12-02T21:14:47,046 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0dfd4169e4c254c51bcc70a472f99ecf, ASSIGN to state=OFFLINE, location=null, table=test, region=0dfd4169e4c254c51bcc70a472f99ecf to restore RIT 2024-12-02T21:14:47,046 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d63da23e8236e18b416716ec838dd43e, ASSIGN to state=OFFLINE, location=null, table=test, region=d63da23e8236e18b416716ec838dd43e to restore RIT 2024-12-02T21:14:47,046 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d0be65f6ffd7ebb4c8b2fee37042136, ASSIGN to state=OFFLINE, location=null, table=test, region=0d0be65f6ffd7ebb4c8b2fee37042136 to restore RIT 2024-12-02T21:14:47,046 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733174117046 2024-12-02T21:14:47,046 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-02T21:14:47,046 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:14:47,048 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-02T21:14:47,049 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-02T21:14:47,049 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0d0be65f6ffd7ebb4c8b2fee37042136, server=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:47,049 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T21:14:47,049 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0d0be65f6ffd7ebb4c8b2fee37042136, server=87c3fdb6c570,40413,1733174078061 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:14:47,054 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85ca3a9a09d75c0941c58f5b83584a8e, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,054 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ed86f04f519cc5dccd1b6d41be3da7d, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,055 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0517912ff18e81317bb5474cefaca025, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,055 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff611163b4ab91c50d940651a0065c34, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,055 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=120cf3776e92a15d3276f11911c4042f, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=5 2024-12-02T21:14:47,055 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e87d99b937228303e28b7ea0373ddc70, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,055 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0dfd4169e4c254c51bcc70a472f99ecf, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,055 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9950441eda7548fa66872811a13f5bf0, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,056 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e039115fcc0d0a54e2d03efed7e05de, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,056 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d60c566b39516dd875aaf2bd7def402, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,056 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a524d88be9ac29b416f0a6c04627fa1c, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,056 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0d0be65f6ffd7ebb4c8b2fee37042136, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,056 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=292fd0b4e0f07516cc25c069181e3111, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,056 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87ba365a3e5d028c9193155af96aeb60, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,057 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4197cd4f7ea2f3d6256be6836e9834da, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,057 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=20545c8ee2d66091a91ff9697d1f18ca, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,057 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a50bae1d5cb3d03a86dc103b09a0f68b, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,057 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=215753e705337ae11b1b38ad77b6a5cd, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,057 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e42e4828e881f12c74fbedcdb6b00ea2, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,057 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d63da23e8236e18b416716ec838dd43e, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,058 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e40e25a9323e1d3dad4ec92c578e2f82, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,058 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a2fb6e9d39bf04774472d3b4c70810a, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,058 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1a4ea0dec946dcfe96ad162d5bcf5949, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,058 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06d131e39cb4f5b02db4ec472d438a26, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,058 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0875ddf9cbd035a3345be0684b07b2e6, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,058 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a36d8f99c2aef77f88827585926ad989, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,058 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-02T21:14:47,059 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733174147059 2024-12-02T21:14:47,059 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733174207059 2024-12-02T21:14:47,059 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 12 msec 2024-12-02T21:14:47,059 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-02T21:14:47,059 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T21:14:47,059 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-02T21:14:47,059 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-02T21:14:47,059 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-02T21:14:47,059 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-02T21:14:47,059 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-02T21:14:47,060 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-02T21:14:47,061 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-02T21:14:47,063 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-02T21:14:47,064 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T21:14:47,064 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T21:14:47,064 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,064 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,064 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,064 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,064 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T21:14:47,064 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,064 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,064 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T21:14:47,064 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,064 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,064 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T21:14:47,064 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T21:14:47,064 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T21:14:47,064 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,064 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T21:14:47,065 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-02T21:14:47,065 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-02T21:14:47,065 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=120cf3776e92a15d3276f11911c4042f, REOPEN/MOVE 2024-12-02T21:14:47,067 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true 2024-12-02T21:14:47,067 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:47,067 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:47,067 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T21:14:47,067 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:47,067 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:47,067 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:47,067 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN 2024-12-02T21:14:47,067 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN 2024-12-02T21:14:47,068 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN 2024-12-02T21:14:47,068 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=87ba365a3e5d028c9193155af96aeb60, ASSIGN 2024-12-02T21:14:47,068 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b, ASSIGN 2024-12-02T21:14:47,068 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0dfd4169e4c254c51bcc70a472f99ecf, ASSIGN 2024-12-02T21:14:47,068 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d63da23e8236e18b416716ec838dd43e, ASSIGN 2024-12-02T21:14:47,068 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d0be65f6ffd7ebb4c8b2fee37042136, ASSIGN 2024-12-02T21:14:47,068 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0dfd4169e4c254c51bcc70a472f99ecf, server=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:47,068 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 87ba365a3e5d028c9193155af96aeb60, server=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:47,068 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0d0be65f6ffd7ebb4c8b2fee37042136, server=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:47,068 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,068 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,068 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,068 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,068 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,068 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,068 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,068 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,068 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN 2024-12-02T21:14:47,068 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,068 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN 2024-12-02T21:14:47,068 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,068 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN 2024-12-02T21:14:47,069 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=87ba365a3e5d028c9193155af96aeb60, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,069 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=87ba365a3e5d028c9193155af96aeb60, ASSIGN 2024-12-02T21:14:47,069 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,069 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b, ASSIGN 2024-12-02T21:14:47,069 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0dfd4169e4c254c51bcc70a472f99ecf, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,069 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0dfd4169e4c254c51bcc70a472f99ecf, ASSIGN 2024-12-02T21:14:47,069 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d63da23e8236e18b416716ec838dd43e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,069 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d63da23e8236e18b416716ec838dd43e, ASSIGN 2024-12-02T21:14:47,069 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d0be65f6ffd7ebb4c8b2fee37042136, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,069 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d0be65f6ffd7ebb4c8b2fee37042136, ASSIGN 2024-12-02T21:14:47,069 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0dfd4169e4c254c51bcc70a472f99ecf, server=87c3fdb6c570,36585,1733174078182 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,069 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 87ba365a3e5d028c9193155af96aeb60, server=87c3fdb6c570,36585,1733174078182 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,069 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0d0be65f6ffd7ebb4c8b2fee37042136, server=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,069 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-12-02T21:14:47,070 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=65 2024-12-02T21:14:47,070 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=87c3fdb6c570,40413,1733174078061, table=hbase:meta, region=1588230740 2024-12-02T21:14:47,070 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 87c3fdb6c570,40413,1733174078061, state=OPEN 2024-12-02T21:14:47,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42045-0x10197e5a0060000, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:47,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36585-0x10197e5a0060002, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:47,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40413-0x10197e5a0060001, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:47,083 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:47,083 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:47,083 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:47,083 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN to state=OFFLINE, location=null, table=test, region=06d131e39cb4f5b02db4ec472d438a26 to restore RIT 2024-12-02T21:14:47,083 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN to state=OFFLINE, location=null, table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949 to restore RIT 2024-12-02T21:14:47,083 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN to state=OFFLINE, location=null, table=test, region=0517912ff18e81317bb5474cefaca025 to restore RIT 2024-12-02T21:14:47,083 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=87ba365a3e5d028c9193155af96aeb60, ASSIGN to state=OFFLINE, location=null, table=test, region=87ba365a3e5d028c9193155af96aeb60 to restore RIT 2024-12-02T21:14:47,083 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b, ASSIGN to state=OFFLINE, location=null, table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b to restore RIT 2024-12-02T21:14:47,083 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0dfd4169e4c254c51bcc70a472f99ecf, ASSIGN to state=OFFLINE, location=null, table=test, region=0dfd4169e4c254c51bcc70a472f99ecf to restore RIT 2024-12-02T21:14:47,084 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d63da23e8236e18b416716ec838dd43e, ASSIGN to state=OFFLINE, location=null, table=test, region=d63da23e8236e18b416716ec838dd43e to restore RIT 2024-12-02T21:14:47,084 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d0be65f6ffd7ebb4c8b2fee37042136, ASSIGN to state=OFFLINE, location=null, table=test, region=0d0be65f6ffd7ebb4c8b2fee37042136 to restore RIT 2024-12-02T21:14:47,084 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733174117084 2024-12-02T21:14:47,084 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-02T21:14:47,084 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:14:47,086 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-02T21:14:47,086 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-02T21:14:47,087 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 87ba365a3e5d028c9193155af96aeb60, server=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:47,087 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T21:14:47,087 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 87ba365a3e5d028c9193155af96aeb60, server=87c3fdb6c570,36585,1733174078182 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:14:47,091 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85ca3a9a09d75c0941c58f5b83584a8e, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,091 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ed86f04f519cc5dccd1b6d41be3da7d, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,091 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0517912ff18e81317bb5474cefaca025, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,091 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff611163b4ab91c50d940651a0065c34, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=120cf3776e92a15d3276f11911c4042f, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=5 2024-12-02T21:14:47,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e87d99b937228303e28b7ea0373ddc70, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0dfd4169e4c254c51bcc70a472f99ecf, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9950441eda7548fa66872811a13f5bf0, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e039115fcc0d0a54e2d03efed7e05de, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d60c566b39516dd875aaf2bd7def402, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a524d88be9ac29b416f0a6c04627fa1c, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0d0be65f6ffd7ebb4c8b2fee37042136, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=292fd0b4e0f07516cc25c069181e3111, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87ba365a3e5d028c9193155af96aeb60, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,093 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4197cd4f7ea2f3d6256be6836e9834da, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,093 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=20545c8ee2d66091a91ff9697d1f18ca, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,093 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a50bae1d5cb3d03a86dc103b09a0f68b, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,093 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=215753e705337ae11b1b38ad77b6a5cd, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,093 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e42e4828e881f12c74fbedcdb6b00ea2, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,093 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d63da23e8236e18b416716ec838dd43e, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,093 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e40e25a9323e1d3dad4ec92c578e2f82, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,093 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a2fb6e9d39bf04774472d3b4c70810a, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,093 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1a4ea0dec946dcfe96ad162d5bcf5949, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,093 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06d131e39cb4f5b02db4ec472d438a26, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,094 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0875ddf9cbd035a3345be0684b07b2e6, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,094 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a36d8f99c2aef77f88827585926ad989, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,094 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-02T21:14:47,094 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733174147094 2024-12-02T21:14:47,094 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733174207094 2024-12-02T21:14:47,094 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-12-02T21:14:47,094 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-02T21:14:47,094 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T21:14:47,094 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-02T21:14:47,094 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-02T21:14:47,094 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-02T21:14:47,094 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-02T21:14:47,094 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-02T21:14:47,095 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-02T21:14:47,095 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-02T21:14:47,097 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-02T21:14:47,097 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T21:14:47,097 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T21:14:47,097 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,097 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,097 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,097 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,097 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T21:14:47,097 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,097 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,097 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T21:14:47,097 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,097 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,097 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T21:14:47,097 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T21:14:47,097 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,097 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T21:14:47,098 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-02T21:14:47,098 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-02T21:14:47,098 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=120cf3776e92a15d3276f11911c4042f, REOPEN/MOVE 2024-12-02T21:14:47,099 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true 2024-12-02T21:14:47,099 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:47,099 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:47,099 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T21:14:47,099 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:47,099 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:47,100 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:47,100 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN 2024-12-02T21:14:47,100 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN 2024-12-02T21:14:47,100 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN 2024-12-02T21:14:47,100 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=87ba365a3e5d028c9193155af96aeb60, ASSIGN 2024-12-02T21:14:47,100 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b, ASSIGN 2024-12-02T21:14:47,100 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0dfd4169e4c254c51bcc70a472f99ecf, ASSIGN 2024-12-02T21:14:47,100 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d63da23e8236e18b416716ec838dd43e, ASSIGN 2024-12-02T21:14:47,100 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d0be65f6ffd7ebb4c8b2fee37042136, ASSIGN 2024-12-02T21:14:47,100 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0dfd4169e4c254c51bcc70a472f99ecf, server=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:47,100 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 87ba365a3e5d028c9193155af96aeb60, server=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:47,100 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,100 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,100 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,100 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,100 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,100 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,100 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,100 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,100 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN 2024-12-02T21:14:47,100 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,100 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN 2024-12-02T21:14:47,100 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,100 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN 2024-12-02T21:14:47,100 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=87ba365a3e5d028c9193155af96aeb60, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,100 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=87ba365a3e5d028c9193155af96aeb60, ASSIGN 2024-12-02T21:14:47,100 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,100 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b, ASSIGN 2024-12-02T21:14:47,100 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0dfd4169e4c254c51bcc70a472f99ecf, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,100 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0dfd4169e4c254c51bcc70a472f99ecf, ASSIGN 2024-12-02T21:14:47,101 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d63da23e8236e18b416716ec838dd43e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,101 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d63da23e8236e18b416716ec838dd43e, ASSIGN 2024-12-02T21:14:47,101 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d0be65f6ffd7ebb4c8b2fee37042136, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,101 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d0be65f6ffd7ebb4c8b2fee37042136, ASSIGN 2024-12-02T21:14:47,101 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0dfd4169e4c254c51bcc70a472f99ecf, server=87c3fdb6c570,36585,1733174078182 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,101 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 87ba365a3e5d028c9193155af96aeb60, server=87c3fdb6c570,36585,1733174078182 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,101 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-12-02T21:14:47,101 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=65 2024-12-02T21:14:47,102 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=87c3fdb6c570,40413,1733174078061, table=hbase:meta, region=1588230740 2024-12-02T21:14:47,102 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 87c3fdb6c570,40413,1733174078061, state=OPEN 2024-12-02T21:14:47,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40413-0x10197e5a0060001, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:47,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36585-0x10197e5a0060002, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:47,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42045-0x10197e5a0060000, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:47,114 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:47,114 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:47,114 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:47,116 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN to state=OFFLINE, location=null, table=test, region=06d131e39cb4f5b02db4ec472d438a26 to restore RIT 2024-12-02T21:14:47,117 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN to state=OFFLINE, location=null, table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949 to restore RIT 2024-12-02T21:14:47,117 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN to state=OFFLINE, location=null, table=test, region=0517912ff18e81317bb5474cefaca025 to restore RIT 2024-12-02T21:14:47,117 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=87ba365a3e5d028c9193155af96aeb60, ASSIGN to state=OFFLINE, location=null, table=test, region=87ba365a3e5d028c9193155af96aeb60 to restore RIT 2024-12-02T21:14:47,117 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b, ASSIGN to state=OFFLINE, location=null, table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b to restore RIT 2024-12-02T21:14:47,117 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0dfd4169e4c254c51bcc70a472f99ecf, ASSIGN to state=OFFLINE, location=null, table=test, region=0dfd4169e4c254c51bcc70a472f99ecf to restore RIT 2024-12-02T21:14:47,117 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d63da23e8236e18b416716ec838dd43e, ASSIGN to state=OFFLINE, location=null, table=test, region=d63da23e8236e18b416716ec838dd43e to restore RIT 2024-12-02T21:14:47,117 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d0be65f6ffd7ebb4c8b2fee37042136, ASSIGN to state=OFFLINE, location=null, table=test, region=0d0be65f6ffd7ebb4c8b2fee37042136 to restore RIT 2024-12-02T21:14:47,118 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733174117118 2024-12-02T21:14:47,118 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-02T21:14:47,118 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:14:47,120 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-02T21:14:47,124 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85ca3a9a09d75c0941c58f5b83584a8e, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,124 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-02T21:14:47,124 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=75, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0dfd4169e4c254c51bcc70a472f99ecf, server=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:47,124 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T21:14:47,124 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ed86f04f519cc5dccd1b6d41be3da7d, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,124 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=75, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0dfd4169e4c254c51bcc70a472f99ecf, server=87c3fdb6c570,36585,1733174078182 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:14:47,124 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0517912ff18e81317bb5474cefaca025, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,124 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff611163b4ab91c50d940651a0065c34, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,125 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=120cf3776e92a15d3276f11911c4042f, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=5 2024-12-02T21:14:47,125 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e87d99b937228303e28b7ea0373ddc70, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,125 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0dfd4169e4c254c51bcc70a472f99ecf, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,125 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9950441eda7548fa66872811a13f5bf0, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,125 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e039115fcc0d0a54e2d03efed7e05de, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,126 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d60c566b39516dd875aaf2bd7def402, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,126 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a524d88be9ac29b416f0a6c04627fa1c, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,126 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0d0be65f6ffd7ebb4c8b2fee37042136, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,126 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=292fd0b4e0f07516cc25c069181e3111, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,126 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87ba365a3e5d028c9193155af96aeb60, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,126 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4197cd4f7ea2f3d6256be6836e9834da, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,126 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=20545c8ee2d66091a91ff9697d1f18ca, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a50bae1d5cb3d03a86dc103b09a0f68b, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=215753e705337ae11b1b38ad77b6a5cd, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e42e4828e881f12c74fbedcdb6b00ea2, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d63da23e8236e18b416716ec838dd43e, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e40e25a9323e1d3dad4ec92c578e2f82, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a2fb6e9d39bf04774472d3b4c70810a, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,127 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1a4ea0dec946dcfe96ad162d5bcf5949, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,128 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06d131e39cb4f5b02db4ec472d438a26, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,128 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0875ddf9cbd035a3345be0684b07b2e6, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,128 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a36d8f99c2aef77f88827585926ad989, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,128 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-02T21:14:47,128 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733174147128 2024-12-02T21:14:47,128 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733174207128 2024-12-02T21:14:47,128 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-12-02T21:14:47,128 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-02T21:14:47,128 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T21:14:47,128 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-02T21:14:47,129 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-02T21:14:47,129 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-02T21:14:47,129 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-02T21:14:47,129 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-02T21:14:47,129 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-02T21:14:47,130 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-02T21:14:47,133 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-02T21:14:47,133 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T21:14:47,133 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T21:14:47,133 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,133 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,133 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,133 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,133 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,133 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,133 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T21:14:47,133 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,133 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,133 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T21:14:47,133 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T21:14:47,133 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,133 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T21:14:47,134 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-02T21:14:47,135 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-02T21:14:47,135 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=120cf3776e92a15d3276f11911c4042f, REOPEN/MOVE 2024-12-02T21:14:47,137 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true 2024-12-02T21:14:47,137 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:47,137 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:47,137 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T21:14:47,137 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:47,137 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:47,137 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:47,137 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN 2024-12-02T21:14:47,137 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN 2024-12-02T21:14:47,137 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN 2024-12-02T21:14:47,137 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=87ba365a3e5d028c9193155af96aeb60, ASSIGN 2024-12-02T21:14:47,137 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b, ASSIGN 2024-12-02T21:14:47,137 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0dfd4169e4c254c51bcc70a472f99ecf, ASSIGN 2024-12-02T21:14:47,138 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d63da23e8236e18b416716ec838dd43e, ASSIGN 2024-12-02T21:14:47,138 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d0be65f6ffd7ebb4c8b2fee37042136, ASSIGN 2024-12-02T21:14:47,138 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0dfd4169e4c254c51bcc70a472f99ecf, server=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:47,138 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,138 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,138 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,138 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,138 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,138 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,138 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,138 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,138 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN 2024-12-02T21:14:47,138 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,138 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN 2024-12-02T21:14:47,138 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,138 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN 2024-12-02T21:14:47,138 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=87ba365a3e5d028c9193155af96aeb60, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,138 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=87ba365a3e5d028c9193155af96aeb60, ASSIGN 2024-12-02T21:14:47,138 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,138 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b, ASSIGN 2024-12-02T21:14:47,139 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0dfd4169e4c254c51bcc70a472f99ecf, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,139 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0dfd4169e4c254c51bcc70a472f99ecf, ASSIGN 2024-12-02T21:14:47,139 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d63da23e8236e18b416716ec838dd43e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,139 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d63da23e8236e18b416716ec838dd43e, ASSIGN 2024-12-02T21:14:47,139 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d0be65f6ffd7ebb4c8b2fee37042136, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,139 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d0be65f6ffd7ebb4c8b2fee37042136, ASSIGN 2024-12-02T21:14:47,139 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0dfd4169e4c254c51bcc70a472f99ecf, server=87c3fdb6c570,36585,1733174078182 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,139 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-12-02T21:14:47,140 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=65 2024-12-02T21:14:47,140 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=87c3fdb6c570,40413,1733174078061, table=hbase:meta, region=1588230740 2024-12-02T21:14:47,140 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 87c3fdb6c570,40413,1733174078061, state=OPEN 2024-12-02T21:14:47,150 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42045-0x10197e5a0060000, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:47,150 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36585-0x10197e5a0060002, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:47,150 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40413-0x10197e5a0060001, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:47,150 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:47,150 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:47,150 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:47,151 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN to state=OFFLINE, location=null, table=test, region=06d131e39cb4f5b02db4ec472d438a26 to restore RIT 2024-12-02T21:14:47,151 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN to state=OFFLINE, location=null, table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949 to restore RIT 2024-12-02T21:14:47,151 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN to state=OFFLINE, location=null, table=test, region=0517912ff18e81317bb5474cefaca025 to restore RIT 2024-12-02T21:14:47,151 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=87ba365a3e5d028c9193155af96aeb60, ASSIGN to state=OFFLINE, location=null, table=test, region=87ba365a3e5d028c9193155af96aeb60 to restore RIT 2024-12-02T21:14:47,151 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b, ASSIGN to state=OFFLINE, location=null, table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b to restore RIT 2024-12-02T21:14:47,151 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0dfd4169e4c254c51bcc70a472f99ecf, ASSIGN to state=OFFLINE, location=null, table=test, region=0dfd4169e4c254c51bcc70a472f99ecf to restore RIT 2024-12-02T21:14:47,151 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d63da23e8236e18b416716ec838dd43e, ASSIGN to state=OFFLINE, location=null, table=test, region=d63da23e8236e18b416716ec838dd43e to restore RIT 2024-12-02T21:14:47,151 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d0be65f6ffd7ebb4c8b2fee37042136, ASSIGN to state=OFFLINE, location=null, table=test, region=0d0be65f6ffd7ebb4c8b2fee37042136 to restore RIT 2024-12-02T21:14:47,151 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733174117151 2024-12-02T21:14:47,151 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-02T21:14:47,151 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:14:47,153 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-02T21:14:47,154 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-02T21:14:47,154 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d0be65f6ffd7ebb4c8b2fee37042136, ASSIGN 2024-12-02T21:14:47,154 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T21:14:47,154 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d0be65f6ffd7ebb4c8b2fee37042136, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:14:47,158 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85ca3a9a09d75c0941c58f5b83584a8e, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ed86f04f519cc5dccd1b6d41be3da7d, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0517912ff18e81317bb5474cefaca025, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff611163b4ab91c50d940651a0065c34, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=120cf3776e92a15d3276f11911c4042f, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=5 2024-12-02T21:14:47,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e87d99b937228303e28b7ea0373ddc70, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,159 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0dfd4169e4c254c51bcc70a472f99ecf, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9950441eda7548fa66872811a13f5bf0, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e039115fcc0d0a54e2d03efed7e05de, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d60c566b39516dd875aaf2bd7def402, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a524d88be9ac29b416f0a6c04627fa1c, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0d0be65f6ffd7ebb4c8b2fee37042136, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=292fd0b4e0f07516cc25c069181e3111, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87ba365a3e5d028c9193155af96aeb60, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,160 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4197cd4f7ea2f3d6256be6836e9834da, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,161 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=20545c8ee2d66091a91ff9697d1f18ca, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,161 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a50bae1d5cb3d03a86dc103b09a0f68b, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,161 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=215753e705337ae11b1b38ad77b6a5cd, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,161 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e42e4828e881f12c74fbedcdb6b00ea2, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,161 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d63da23e8236e18b416716ec838dd43e, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,161 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e40e25a9323e1d3dad4ec92c578e2f82, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,161 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a2fb6e9d39bf04774472d3b4c70810a, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,161 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1a4ea0dec946dcfe96ad162d5bcf5949, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,161 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06d131e39cb4f5b02db4ec472d438a26, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,161 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0875ddf9cbd035a3345be0684b07b2e6, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,162 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a36d8f99c2aef77f88827585926ad989, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,162 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-02T21:14:47,162 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733174147162 2024-12-02T21:14:47,162 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733174207162 2024-12-02T21:14:47,162 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-12-02T21:14:47,162 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-02T21:14:47,162 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T21:14:47,162 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-02T21:14:47,162 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-02T21:14:47,162 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-02T21:14:47,162 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-02T21:14:47,162 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-02T21:14:47,163 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-02T21:14:47,163 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-02T21:14:47,165 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-02T21:14:47,165 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T21:14:47,165 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T21:14:47,165 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,165 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,165 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,165 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,165 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,165 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,165 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,165 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,165 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T21:14:47,165 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T21:14:47,165 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,165 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T21:14:47,166 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-02T21:14:47,166 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-02T21:14:47,166 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=120cf3776e92a15d3276f11911c4042f, REOPEN/MOVE 2024-12-02T21:14:47,168 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true 2024-12-02T21:14:47,168 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:47,168 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:47,168 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T21:14:47,168 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:47,168 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:47,168 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:47,168 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN 2024-12-02T21:14:47,168 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN 2024-12-02T21:14:47,168 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN 2024-12-02T21:14:47,168 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=87ba365a3e5d028c9193155af96aeb60, ASSIGN 2024-12-02T21:14:47,168 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b, ASSIGN 2024-12-02T21:14:47,168 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0dfd4169e4c254c51bcc70a472f99ecf, ASSIGN 2024-12-02T21:14:47,168 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d63da23e8236e18b416716ec838dd43e, ASSIGN 2024-12-02T21:14:47,168 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d0be65f6ffd7ebb4c8b2fee37042136, ASSIGN 2024-12-02T21:14:47,169 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,169 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,169 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,169 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,169 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,169 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,169 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,169 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,169 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN 2024-12-02T21:14:47,169 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,169 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN 2024-12-02T21:14:47,169 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,169 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN 2024-12-02T21:14:47,169 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=87ba365a3e5d028c9193155af96aeb60, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,169 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=87ba365a3e5d028c9193155af96aeb60, ASSIGN 2024-12-02T21:14:47,169 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,169 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b, ASSIGN 2024-12-02T21:14:47,169 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0dfd4169e4c254c51bcc70a472f99ecf, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,169 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0dfd4169e4c254c51bcc70a472f99ecf, ASSIGN 2024-12-02T21:14:47,169 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d63da23e8236e18b416716ec838dd43e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,169 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d63da23e8236e18b416716ec838dd43e, ASSIGN 2024-12-02T21:14:47,169 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d0be65f6ffd7ebb4c8b2fee37042136, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,169 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d0be65f6ffd7ebb4c8b2fee37042136, ASSIGN 2024-12-02T21:14:47,169 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-12-02T21:14:47,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=65 2024-12-02T21:14:47,170 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=87c3fdb6c570,40413,1733174078061, table=hbase:meta, region=1588230740 2024-12-02T21:14:47,171 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 87c3fdb6c570,40413,1733174078061, state=OPEN 2024-12-02T21:14:47,181 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40413-0x10197e5a0060001, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:47,181 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36585-0x10197e5a0060002, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:47,181 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42045-0x10197e5a0060000, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:47,182 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:47,182 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:47,182 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:47,182 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN to state=OFFLINE, location=null, table=test, region=06d131e39cb4f5b02db4ec472d438a26 to restore RIT 2024-12-02T21:14:47,182 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN to state=OFFLINE, location=null, table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949 to restore RIT 2024-12-02T21:14:47,182 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN to state=OFFLINE, location=null, table=test, region=0517912ff18e81317bb5474cefaca025 to restore RIT 2024-12-02T21:14:47,182 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=87ba365a3e5d028c9193155af96aeb60, ASSIGN to state=OFFLINE, location=null, table=test, region=87ba365a3e5d028c9193155af96aeb60 to restore RIT 2024-12-02T21:14:47,182 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b, ASSIGN to state=OFFLINE, location=null, table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b to restore RIT 2024-12-02T21:14:47,182 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0dfd4169e4c254c51bcc70a472f99ecf, ASSIGN to state=OFFLINE, location=null, table=test, region=0dfd4169e4c254c51bcc70a472f99ecf to restore RIT 2024-12-02T21:14:47,183 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d63da23e8236e18b416716ec838dd43e, ASSIGN to state=OFFLINE, location=null, table=test, region=d63da23e8236e18b416716ec838dd43e to restore RIT 2024-12-02T21:14:47,183 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0d0be65f6ffd7ebb4c8b2fee37042136, ASSIGN to state=OFFLINE, location=null, table=test, region=0d0be65f6ffd7ebb4c8b2fee37042136 to restore RIT 2024-12-02T21:14:47,183 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733174117183 2024-12-02T21:14:47,183 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-02T21:14:47,183 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:14:47,184 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-02T21:14:47,185 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-02T21:14:47,185 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d63da23e8236e18b416716ec838dd43e, ASSIGN 2024-12-02T21:14:47,185 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T21:14:47,185 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d63da23e8236e18b416716ec838dd43e, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:14:47,188 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85ca3a9a09d75c0941c58f5b83584a8e, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,188 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ed86f04f519cc5dccd1b6d41be3da7d, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,188 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0517912ff18e81317bb5474cefaca025, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,188 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff611163b4ab91c50d940651a0065c34, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,189 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=120cf3776e92a15d3276f11911c4042f, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=5 2024-12-02T21:14:47,189 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e87d99b937228303e28b7ea0373ddc70, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,189 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0dfd4169e4c254c51bcc70a472f99ecf, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,189 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9950441eda7548fa66872811a13f5bf0, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,189 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e039115fcc0d0a54e2d03efed7e05de, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,189 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d60c566b39516dd875aaf2bd7def402, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,190 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a524d88be9ac29b416f0a6c04627fa1c, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,190 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0d0be65f6ffd7ebb4c8b2fee37042136, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,190 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=292fd0b4e0f07516cc25c069181e3111, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,190 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87ba365a3e5d028c9193155af96aeb60, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,190 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4197cd4f7ea2f3d6256be6836e9834da, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,190 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=20545c8ee2d66091a91ff9697d1f18ca, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,191 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a50bae1d5cb3d03a86dc103b09a0f68b, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,191 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=215753e705337ae11b1b38ad77b6a5cd, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,191 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e42e4828e881f12c74fbedcdb6b00ea2, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,191 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d63da23e8236e18b416716ec838dd43e, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,191 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e40e25a9323e1d3dad4ec92c578e2f82, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,192 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a2fb6e9d39bf04774472d3b4c70810a, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,192 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1a4ea0dec946dcfe96ad162d5bcf5949, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,192 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06d131e39cb4f5b02db4ec472d438a26, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,192 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0875ddf9cbd035a3345be0684b07b2e6, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,192 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a36d8f99c2aef77f88827585926ad989, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,192 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-02T21:14:47,192 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733174147192 2024-12-02T21:14:47,192 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733174207192 2024-12-02T21:14:47,192 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-12-02T21:14:47,193 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-02T21:14:47,193 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T21:14:47,193 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-02T21:14:47,193 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-02T21:14:47,193 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-02T21:14:47,193 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-02T21:14:47,193 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-02T21:14:47,194 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-02T21:14:47,194 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-02T21:14:47,196 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-02T21:14:47,196 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T21:14:47,196 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T21:14:47,197 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,197 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,197 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,197 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,197 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,197 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,197 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,197 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T21:14:47,197 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T21:14:47,197 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,197 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T21:14:47,197 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-02T21:14:47,198 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-02T21:14:47,198 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=120cf3776e92a15d3276f11911c4042f, REOPEN/MOVE 2024-12-02T21:14:47,199 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true 2024-12-02T21:14:47,199 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:47,199 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:47,199 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T21:14:47,199 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:47,199 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:47,200 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:47,200 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN 2024-12-02T21:14:47,200 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN 2024-12-02T21:14:47,200 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN 2024-12-02T21:14:47,200 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=87ba365a3e5d028c9193155af96aeb60, ASSIGN 2024-12-02T21:14:47,200 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b, ASSIGN 2024-12-02T21:14:47,200 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0dfd4169e4c254c51bcc70a472f99ecf, ASSIGN 2024-12-02T21:14:47,200 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d63da23e8236e18b416716ec838dd43e, ASSIGN 2024-12-02T21:14:47,200 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,200 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,200 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,200 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,200 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,200 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,200 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,201 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,201 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN 2024-12-02T21:14:47,201 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,201 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN 2024-12-02T21:14:47,201 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,201 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN 2024-12-02T21:14:47,201 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=87ba365a3e5d028c9193155af96aeb60, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,201 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=87ba365a3e5d028c9193155af96aeb60, ASSIGN 2024-12-02T21:14:47,201 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,201 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b, ASSIGN 2024-12-02T21:14:47,201 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0dfd4169e4c254c51bcc70a472f99ecf, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,201 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0dfd4169e4c254c51bcc70a472f99ecf, ASSIGN 2024-12-02T21:14:47,201 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d63da23e8236e18b416716ec838dd43e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,201 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d63da23e8236e18b416716ec838dd43e, ASSIGN 2024-12-02T21:14:47,201 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-12-02T21:14:47,202 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=65 2024-12-02T21:14:47,202 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=87c3fdb6c570,40413,1733174078061, table=hbase:meta, region=1588230740 2024-12-02T21:14:47,203 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 87c3fdb6c570,40413,1733174078061, state=OPEN 2024-12-02T21:14:47,213 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36585-0x10197e5a0060002, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:47,213 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42045-0x10197e5a0060000, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:47,213 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40413-0x10197e5a0060001, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:47,213 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:47,213 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:47,213 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:47,214 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN to state=OFFLINE, location=null, table=test, region=06d131e39cb4f5b02db4ec472d438a26 to restore RIT 2024-12-02T21:14:47,214 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN to state=OFFLINE, location=null, table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949 to restore RIT 2024-12-02T21:14:47,214 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN to state=OFFLINE, location=null, table=test, region=0517912ff18e81317bb5474cefaca025 to restore RIT 2024-12-02T21:14:47,214 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=87ba365a3e5d028c9193155af96aeb60, ASSIGN to state=OFFLINE, location=null, table=test, region=87ba365a3e5d028c9193155af96aeb60 to restore RIT 2024-12-02T21:14:47,214 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b, ASSIGN to state=OFFLINE, location=null, table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b to restore RIT 2024-12-02T21:14:47,215 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0dfd4169e4c254c51bcc70a472f99ecf, ASSIGN to state=OFFLINE, location=null, table=test, region=0dfd4169e4c254c51bcc70a472f99ecf to restore RIT 2024-12-02T21:14:47,215 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d63da23e8236e18b416716ec838dd43e, ASSIGN to state=OFFLINE, location=null, table=test, region=d63da23e8236e18b416716ec838dd43e to restore RIT 2024-12-02T21:14:47,215 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733174117215 2024-12-02T21:14:47,215 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-02T21:14:47,215 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:14:47,217 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-02T21:14:47,218 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-02T21:14:47,218 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0dfd4169e4c254c51bcc70a472f99ecf, ASSIGN 2024-12-02T21:14:47,218 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T21:14:47,218 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0dfd4169e4c254c51bcc70a472f99ecf, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:14:47,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85ca3a9a09d75c0941c58f5b83584a8e, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ed86f04f519cc5dccd1b6d41be3da7d, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0517912ff18e81317bb5474cefaca025, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff611163b4ab91c50d940651a0065c34, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=120cf3776e92a15d3276f11911c4042f, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=5 2024-12-02T21:14:47,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e87d99b937228303e28b7ea0373ddc70, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0dfd4169e4c254c51bcc70a472f99ecf, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9950441eda7548fa66872811a13f5bf0, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e039115fcc0d0a54e2d03efed7e05de, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d60c566b39516dd875aaf2bd7def402, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a524d88be9ac29b416f0a6c04627fa1c, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0d0be65f6ffd7ebb4c8b2fee37042136, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=292fd0b4e0f07516cc25c069181e3111, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87ba365a3e5d028c9193155af96aeb60, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4197cd4f7ea2f3d6256be6836e9834da, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=20545c8ee2d66091a91ff9697d1f18ca, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,225 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a50bae1d5cb3d03a86dc103b09a0f68b, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,225 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=215753e705337ae11b1b38ad77b6a5cd, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,225 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e42e4828e881f12c74fbedcdb6b00ea2, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,225 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d63da23e8236e18b416716ec838dd43e, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,225 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e40e25a9323e1d3dad4ec92c578e2f82, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,225 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a2fb6e9d39bf04774472d3b4c70810a, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,226 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1a4ea0dec946dcfe96ad162d5bcf5949, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,226 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06d131e39cb4f5b02db4ec472d438a26, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,226 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0875ddf9cbd035a3345be0684b07b2e6, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,226 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a36d8f99c2aef77f88827585926ad989, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,226 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-02T21:14:47,226 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733174147226 2024-12-02T21:14:47,226 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733174207226 2024-12-02T21:14:47,226 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-12-02T21:14:47,226 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-02T21:14:47,226 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T21:14:47,227 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-02T21:14:47,227 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-02T21:14:47,227 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-02T21:14:47,227 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-02T21:14:47,227 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-02T21:14:47,227 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-02T21:14:47,228 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-02T21:14:47,230 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-02T21:14:47,230 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T21:14:47,230 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T21:14:47,230 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,230 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,230 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,230 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,230 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,231 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,231 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T21:14:47,231 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T21:14:47,231 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,231 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T21:14:47,231 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-02T21:14:47,232 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-02T21:14:47,232 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=120cf3776e92a15d3276f11911c4042f, REOPEN/MOVE 2024-12-02T21:14:47,233 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true 2024-12-02T21:14:47,233 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:47,234 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:47,234 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T21:14:47,234 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:47,234 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:47,234 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:47,234 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN 2024-12-02T21:14:47,234 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN 2024-12-02T21:14:47,234 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN 2024-12-02T21:14:47,234 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=87ba365a3e5d028c9193155af96aeb60, ASSIGN 2024-12-02T21:14:47,234 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b, ASSIGN 2024-12-02T21:14:47,234 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0dfd4169e4c254c51bcc70a472f99ecf, ASSIGN 2024-12-02T21:14:47,234 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,234 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,234 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,234 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,234 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,234 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,234 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,234 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,234 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN 2024-12-02T21:14:47,234 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,234 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN 2024-12-02T21:14:47,234 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,235 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN 2024-12-02T21:14:47,235 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=87ba365a3e5d028c9193155af96aeb60, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,235 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=87ba365a3e5d028c9193155af96aeb60, ASSIGN 2024-12-02T21:14:47,235 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,235 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b, ASSIGN 2024-12-02T21:14:47,235 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0dfd4169e4c254c51bcc70a472f99ecf, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,235 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0dfd4169e4c254c51bcc70a472f99ecf, ASSIGN 2024-12-02T21:14:47,235 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-12-02T21:14:47,236 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=65 2024-12-02T21:14:47,236 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=87c3fdb6c570,40413,1733174078061, table=hbase:meta, region=1588230740 2024-12-02T21:14:47,236 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 87c3fdb6c570,40413,1733174078061, state=OPEN 2024-12-02T21:14:47,245 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36585-0x10197e5a0060002, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:47,245 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40413-0x10197e5a0060001, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:47,245 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42045-0x10197e5a0060000, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:47,245 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:47,245 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:47,245 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:47,245 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN to state=OFFLINE, location=null, table=test, region=06d131e39cb4f5b02db4ec472d438a26 to restore RIT 2024-12-02T21:14:47,245 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN to state=OFFLINE, location=null, table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949 to restore RIT 2024-12-02T21:14:47,245 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN to state=OFFLINE, location=null, table=test, region=0517912ff18e81317bb5474cefaca025 to restore RIT 2024-12-02T21:14:47,245 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=87ba365a3e5d028c9193155af96aeb60, ASSIGN to state=OFFLINE, location=null, table=test, region=87ba365a3e5d028c9193155af96aeb60 to restore RIT 2024-12-02T21:14:47,246 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b, ASSIGN to state=OFFLINE, location=null, table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b to restore RIT 2024-12-02T21:14:47,246 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0dfd4169e4c254c51bcc70a472f99ecf, ASSIGN to state=OFFLINE, location=null, table=test, region=0dfd4169e4c254c51bcc70a472f99ecf to restore RIT 2024-12-02T21:14:47,246 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733174117246 2024-12-02T21:14:47,246 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-02T21:14:47,246 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:14:47,247 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-02T21:14:47,248 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-02T21:14:47,248 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b, ASSIGN 2024-12-02T21:14:47,248 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T21:14:47,248 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:14:47,251 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85ca3a9a09d75c0941c58f5b83584a8e, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ed86f04f519cc5dccd1b6d41be3da7d, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0517912ff18e81317bb5474cefaca025, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff611163b4ab91c50d940651a0065c34, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=120cf3776e92a15d3276f11911c4042f, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=5 2024-12-02T21:14:47,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e87d99b937228303e28b7ea0373ddc70, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0dfd4169e4c254c51bcc70a472f99ecf, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9950441eda7548fa66872811a13f5bf0, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e039115fcc0d0a54e2d03efed7e05de, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d60c566b39516dd875aaf2bd7def402, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a524d88be9ac29b416f0a6c04627fa1c, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0d0be65f6ffd7ebb4c8b2fee37042136, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=292fd0b4e0f07516cc25c069181e3111, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87ba365a3e5d028c9193155af96aeb60, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4197cd4f7ea2f3d6256be6836e9834da, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=20545c8ee2d66091a91ff9697d1f18ca, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a50bae1d5cb3d03a86dc103b09a0f68b, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=215753e705337ae11b1b38ad77b6a5cd, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e42e4828e881f12c74fbedcdb6b00ea2, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d63da23e8236e18b416716ec838dd43e, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e40e25a9323e1d3dad4ec92c578e2f82, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a2fb6e9d39bf04774472d3b4c70810a, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1a4ea0dec946dcfe96ad162d5bcf5949, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06d131e39cb4f5b02db4ec472d438a26, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0875ddf9cbd035a3345be0684b07b2e6, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a36d8f99c2aef77f88827585926ad989, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,255 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-02T21:14:47,255 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733174147255 2024-12-02T21:14:47,255 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733174207255 2024-12-02T21:14:47,255 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-12-02T21:14:47,255 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-02T21:14:47,255 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T21:14:47,255 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-02T21:14:47,255 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-02T21:14:47,255 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-02T21:14:47,255 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-02T21:14:47,255 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-02T21:14:47,256 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-02T21:14:47,256 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-02T21:14:47,259 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-02T21:14:47,259 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T21:14:47,259 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T21:14:47,259 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,259 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,259 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,259 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,259 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,259 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T21:14:47,259 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T21:14:47,259 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,259 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T21:14:47,260 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-02T21:14:47,260 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-02T21:14:47,261 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=120cf3776e92a15d3276f11911c4042f, REOPEN/MOVE 2024-12-02T21:14:47,262 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true 2024-12-02T21:14:47,262 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:47,262 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:47,263 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T21:14:47,263 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:47,263 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:47,263 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:47,263 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN 2024-12-02T21:14:47,263 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN 2024-12-02T21:14:47,263 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN 2024-12-02T21:14:47,263 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=87ba365a3e5d028c9193155af96aeb60, ASSIGN 2024-12-02T21:14:47,263 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b, ASSIGN 2024-12-02T21:14:47,263 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,263 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,263 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,263 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,263 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,263 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,264 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,264 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,264 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN 2024-12-02T21:14:47,264 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,264 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN 2024-12-02T21:14:47,264 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,264 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN 2024-12-02T21:14:47,264 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=87ba365a3e5d028c9193155af96aeb60, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,264 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=87ba365a3e5d028c9193155af96aeb60, ASSIGN 2024-12-02T21:14:47,264 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,264 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b, ASSIGN 2024-12-02T21:14:47,264 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-12-02T21:14:47,265 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=65 2024-12-02T21:14:47,266 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=87c3fdb6c570,40413,1733174078061, table=hbase:meta, region=1588230740 2024-12-02T21:14:47,266 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 87c3fdb6c570,40413,1733174078061, state=OPEN 2024-12-02T21:14:47,276 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40413-0x10197e5a0060001, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:47,276 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42045-0x10197e5a0060000, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:47,276 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36585-0x10197e5a0060002, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:47,276 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:47,276 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:47,276 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:47,277 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN to state=OFFLINE, location=null, table=test, region=06d131e39cb4f5b02db4ec472d438a26 to restore RIT 2024-12-02T21:14:47,277 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN to state=OFFLINE, location=null, table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949 to restore RIT 2024-12-02T21:14:47,277 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN to state=OFFLINE, location=null, table=test, region=0517912ff18e81317bb5474cefaca025 to restore RIT 2024-12-02T21:14:47,277 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=87ba365a3e5d028c9193155af96aeb60, ASSIGN to state=OFFLINE, location=null, table=test, region=87ba365a3e5d028c9193155af96aeb60 to restore RIT 2024-12-02T21:14:47,277 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b, ASSIGN to state=OFFLINE, location=null, table=test, region=a50bae1d5cb3d03a86dc103b09a0f68b to restore RIT 2024-12-02T21:14:47,278 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733174117278 2024-12-02T21:14:47,278 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-02T21:14:47,278 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:14:47,279 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-02T21:14:47,281 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-02T21:14:47,281 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=87ba365a3e5d028c9193155af96aeb60, ASSIGN 2024-12-02T21:14:47,281 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T21:14:47,281 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=87ba365a3e5d028c9193155af96aeb60, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:14:47,283 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85ca3a9a09d75c0941c58f5b83584a8e, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,283 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ed86f04f519cc5dccd1b6d41be3da7d, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,283 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0517912ff18e81317bb5474cefaca025, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,283 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff611163b4ab91c50d940651a0065c34, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,283 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=120cf3776e92a15d3276f11911c4042f, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=5 2024-12-02T21:14:47,284 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e87d99b937228303e28b7ea0373ddc70, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,284 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0dfd4169e4c254c51bcc70a472f99ecf, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,284 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9950441eda7548fa66872811a13f5bf0, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,284 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e039115fcc0d0a54e2d03efed7e05de, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,284 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d60c566b39516dd875aaf2bd7def402, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,284 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a524d88be9ac29b416f0a6c04627fa1c, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,284 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0d0be65f6ffd7ebb4c8b2fee37042136, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,284 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=292fd0b4e0f07516cc25c069181e3111, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,284 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87ba365a3e5d028c9193155af96aeb60, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,284 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4197cd4f7ea2f3d6256be6836e9834da, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,284 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=20545c8ee2d66091a91ff9697d1f18ca, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,285 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a50bae1d5cb3d03a86dc103b09a0f68b, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,285 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=215753e705337ae11b1b38ad77b6a5cd, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,285 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e42e4828e881f12c74fbedcdb6b00ea2, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,285 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d63da23e8236e18b416716ec838dd43e, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,285 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e40e25a9323e1d3dad4ec92c578e2f82, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,285 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a2fb6e9d39bf04774472d3b4c70810a, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,285 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1a4ea0dec946dcfe96ad162d5bcf5949, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,285 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06d131e39cb4f5b02db4ec472d438a26, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,285 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0875ddf9cbd035a3345be0684b07b2e6, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,285 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a36d8f99c2aef77f88827585926ad989, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,285 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-02T21:14:47,285 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733174147285 2024-12-02T21:14:47,285 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733174207285 2024-12-02T21:14:47,285 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-12-02T21:14:47,285 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-02T21:14:47,286 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T21:14:47,286 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-02T21:14:47,286 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-02T21:14:47,286 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-02T21:14:47,286 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-02T21:14:47,286 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-02T21:14:47,286 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-02T21:14:47,287 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-02T21:14:47,288 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-02T21:14:47,288 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T21:14:47,288 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T21:14:47,288 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,288 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,289 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,289 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,289 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T21:14:47,289 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T21:14:47,289 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,289 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T21:14:47,289 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-02T21:14:47,289 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-02T21:14:47,289 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=120cf3776e92a15d3276f11911c4042f, REOPEN/MOVE 2024-12-02T21:14:47,290 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true 2024-12-02T21:14:47,290 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:47,290 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:47,290 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T21:14:47,290 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:47,290 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:47,290 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:47,290 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN 2024-12-02T21:14:47,290 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN 2024-12-02T21:14:47,290 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN 2024-12-02T21:14:47,290 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=87ba365a3e5d028c9193155af96aeb60, ASSIGN 2024-12-02T21:14:47,290 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,291 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,291 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,291 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,291 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,291 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,291 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,291 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,291 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN 2024-12-02T21:14:47,291 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,291 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN 2024-12-02T21:14:47,291 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,291 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN 2024-12-02T21:14:47,291 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=87ba365a3e5d028c9193155af96aeb60, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,291 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=87ba365a3e5d028c9193155af96aeb60, ASSIGN 2024-12-02T21:14:47,291 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-02T21:14:47,292 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=65 2024-12-02T21:14:47,292 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=87c3fdb6c570,40413,1733174078061, table=hbase:meta, region=1588230740 2024-12-02T21:14:47,292 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 87c3fdb6c570,40413,1733174078061, state=OPEN 2024-12-02T21:14:47,294 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42045 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:14:47,295 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42045 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:14:47,304 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36585-0x10197e5a0060002, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:47,304 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42045-0x10197e5a0060000, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:47,304 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40413-0x10197e5a0060001, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:47,304 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:47,304 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:47,304 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:47,304 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN to state=OFFLINE, location=null, table=test, region=06d131e39cb4f5b02db4ec472d438a26 to restore RIT 2024-12-02T21:14:47,305 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN to state=OFFLINE, location=null, table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949 to restore RIT 2024-12-02T21:14:47,305 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN to state=OFFLINE, location=null, table=test, region=0517912ff18e81317bb5474cefaca025 to restore RIT 2024-12-02T21:14:47,305 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=87ba365a3e5d028c9193155af96aeb60, ASSIGN to state=OFFLINE, location=null, table=test, region=87ba365a3e5d028c9193155af96aeb60 to restore RIT 2024-12-02T21:14:47,305 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733174117305 2024-12-02T21:14:47,305 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-02T21:14:47,305 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:14:47,306 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-02T21:14:47,307 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-02T21:14:47,307 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN 2024-12-02T21:14:47,307 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T21:14:47,307 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:14:47,310 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85ca3a9a09d75c0941c58f5b83584a8e, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,310 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ed86f04f519cc5dccd1b6d41be3da7d, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,311 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0517912ff18e81317bb5474cefaca025, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,311 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff611163b4ab91c50d940651a0065c34, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,311 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=120cf3776e92a15d3276f11911c4042f, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=5 2024-12-02T21:14:47,311 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e87d99b937228303e28b7ea0373ddc70, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,311 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0dfd4169e4c254c51bcc70a472f99ecf, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,311 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9950441eda7548fa66872811a13f5bf0, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,311 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e039115fcc0d0a54e2d03efed7e05de, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,311 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d60c566b39516dd875aaf2bd7def402, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,312 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a524d88be9ac29b416f0a6c04627fa1c, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,312 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0d0be65f6ffd7ebb4c8b2fee37042136, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,312 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=292fd0b4e0f07516cc25c069181e3111, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,312 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87ba365a3e5d028c9193155af96aeb60, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,312 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4197cd4f7ea2f3d6256be6836e9834da, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,312 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=20545c8ee2d66091a91ff9697d1f18ca, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,313 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a50bae1d5cb3d03a86dc103b09a0f68b, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,313 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=215753e705337ae11b1b38ad77b6a5cd, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,313 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e42e4828e881f12c74fbedcdb6b00ea2, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,313 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d63da23e8236e18b416716ec838dd43e, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,313 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e40e25a9323e1d3dad4ec92c578e2f82, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,313 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a2fb6e9d39bf04774472d3b4c70810a, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,313 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1a4ea0dec946dcfe96ad162d5bcf5949, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,313 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06d131e39cb4f5b02db4ec472d438a26, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,314 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0875ddf9cbd035a3345be0684b07b2e6, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,314 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a36d8f99c2aef77f88827585926ad989, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,314 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-02T21:14:47,314 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733174147314 2024-12-02T21:14:47,314 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733174207314 2024-12-02T21:14:47,314 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-12-02T21:14:47,314 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-02T21:14:47,314 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T21:14:47,314 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-02T21:14:47,314 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-02T21:14:47,315 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-02T21:14:47,315 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-02T21:14:47,315 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-02T21:14:47,315 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-02T21:14:47,316 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-02T21:14:47,318 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-02T21:14:47,318 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T21:14:47,318 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T21:14:47,318 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,319 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,319 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,319 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T21:14:47,319 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T21:14:47,319 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,319 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T21:14:47,319 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-02T21:14:47,319 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-02T21:14:47,320 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=120cf3776e92a15d3276f11911c4042f, REOPEN/MOVE 2024-12-02T21:14:47,321 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true 2024-12-02T21:14:47,321 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:47,321 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:47,321 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T21:14:47,321 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:47,321 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:47,321 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:47,321 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN 2024-12-02T21:14:47,321 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN 2024-12-02T21:14:47,321 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN 2024-12-02T21:14:47,321 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,321 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,321 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,321 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,321 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,321 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,322 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,322 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,322 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN 2024-12-02T21:14:47,322 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,322 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN 2024-12-02T21:14:47,322 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,322 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN 2024-12-02T21:14:47,322 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-12-02T21:14:47,323 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=65 2024-12-02T21:14:47,323 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=87c3fdb6c570,40413,1733174078061, table=hbase:meta, region=1588230740 2024-12-02T21:14:47,323 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 87c3fdb6c570,40413,1733174078061, state=OPEN 2024-12-02T21:14:47,335 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40413-0x10197e5a0060001, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:47,335 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36585-0x10197e5a0060002, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:47,335 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42045-0x10197e5a0060000, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:47,335 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:47,335 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:47,335 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:47,336 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN to state=OFFLINE, location=null, table=test, region=06d131e39cb4f5b02db4ec472d438a26 to restore RIT 2024-12-02T21:14:47,336 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN to state=OFFLINE, location=null, table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949 to restore RIT 2024-12-02T21:14:47,336 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0517912ff18e81317bb5474cefaca025, ASSIGN to state=OFFLINE, location=null, table=test, region=0517912ff18e81317bb5474cefaca025 to restore RIT 2024-12-02T21:14:47,336 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733174117336 2024-12-02T21:14:47,336 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-02T21:14:47,336 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:14:47,337 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-02T21:14:47,338 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-02T21:14:47,338 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN 2024-12-02T21:14:47,338 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T21:14:47,339 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:14:47,341 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85ca3a9a09d75c0941c58f5b83584a8e, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,341 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ed86f04f519cc5dccd1b6d41be3da7d, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,341 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0517912ff18e81317bb5474cefaca025, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,342 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff611163b4ab91c50d940651a0065c34, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,342 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=120cf3776e92a15d3276f11911c4042f, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=5 2024-12-02T21:14:47,342 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e87d99b937228303e28b7ea0373ddc70, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,342 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0dfd4169e4c254c51bcc70a472f99ecf, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,342 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9950441eda7548fa66872811a13f5bf0, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,342 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e039115fcc0d0a54e2d03efed7e05de, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,342 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d60c566b39516dd875aaf2bd7def402, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,342 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a524d88be9ac29b416f0a6c04627fa1c, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,343 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0d0be65f6ffd7ebb4c8b2fee37042136, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,343 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=292fd0b4e0f07516cc25c069181e3111, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,343 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87ba365a3e5d028c9193155af96aeb60, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,343 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4197cd4f7ea2f3d6256be6836e9834da, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,343 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=20545c8ee2d66091a91ff9697d1f18ca, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,343 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a50bae1d5cb3d03a86dc103b09a0f68b, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,343 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=215753e705337ae11b1b38ad77b6a5cd, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,343 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e42e4828e881f12c74fbedcdb6b00ea2, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,344 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d63da23e8236e18b416716ec838dd43e, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,344 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e40e25a9323e1d3dad4ec92c578e2f82, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,344 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a2fb6e9d39bf04774472d3b4c70810a, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,344 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1a4ea0dec946dcfe96ad162d5bcf5949, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,344 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06d131e39cb4f5b02db4ec472d438a26, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,344 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0875ddf9cbd035a3345be0684b07b2e6, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,344 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a36d8f99c2aef77f88827585926ad989, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,344 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-02T21:14:47,344 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733174147344 2024-12-02T21:14:47,345 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733174207344 2024-12-02T21:14:47,345 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-12-02T21:14:47,345 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-02T21:14:47,345 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T21:14:47,345 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-02T21:14:47,345 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-02T21:14:47,345 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-02T21:14:47,345 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-02T21:14:47,345 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-02T21:14:47,345 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-02T21:14:47,346 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-02T21:14:47,348 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-02T21:14:47,348 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T21:14:47,348 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T21:14:47,348 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,348 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,348 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T21:14:47,348 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T21:14:47,349 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,349 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T21:14:47,349 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-02T21:14:47,349 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-02T21:14:47,350 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=120cf3776e92a15d3276f11911c4042f, REOPEN/MOVE 2024-12-02T21:14:47,351 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true 2024-12-02T21:14:47,351 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:47,351 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:47,351 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T21:14:47,351 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:47,351 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:47,352 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:47,352 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN 2024-12-02T21:14:47,352 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN 2024-12-02T21:14:47,352 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,352 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,352 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,352 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,352 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,352 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,352 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,352 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,352 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN 2024-12-02T21:14:47,352 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,352 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN 2024-12-02T21:14:47,353 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-12-02T21:14:47,353 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=65 2024-12-02T21:14:47,353 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=87c3fdb6c570,40413,1733174078061, table=hbase:meta, region=1588230740 2024-12-02T21:14:47,354 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 87c3fdb6c570,40413,1733174078061, state=OPEN 2024-12-02T21:14:47,397 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36585-0x10197e5a0060002, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:47,397 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40413-0x10197e5a0060001, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:47,397 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42045-0x10197e5a0060000, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:47,397 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:47,397 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:47,397 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:47,398 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN to state=OFFLINE, location=null, table=test, region=06d131e39cb4f5b02db4ec472d438a26 to restore RIT 2024-12-02T21:14:47,398 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949, ASSIGN to state=OFFLINE, location=null, table=test, region=1a4ea0dec946dcfe96ad162d5bcf5949 to restore RIT 2024-12-02T21:14:47,398 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733174117398 2024-12-02T21:14:47,398 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-02T21:14:47,398 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:14:47,399 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-02T21:14:47,403 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85ca3a9a09d75c0941c58f5b83584a8e, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,403 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ed86f04f519cc5dccd1b6d41be3da7d, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,403 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0517912ff18e81317bb5474cefaca025, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,403 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff611163b4ab91c50d940651a0065c34, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,403 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-02T21:14:47,404 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=120cf3776e92a15d3276f11911c4042f, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=5 2024-12-02T21:14:47,404 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN 2024-12-02T21:14:47,404 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T21:14:47,404 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e87d99b937228303e28b7ea0373ddc70, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,404 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0dfd4169e4c254c51bcc70a472f99ecf, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,404 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9950441eda7548fa66872811a13f5bf0, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,404 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:14:47,404 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e039115fcc0d0a54e2d03efed7e05de, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,404 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d60c566b39516dd875aaf2bd7def402, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,404 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a524d88be9ac29b416f0a6c04627fa1c, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,404 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0d0be65f6ffd7ebb4c8b2fee37042136, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,404 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=292fd0b4e0f07516cc25c069181e3111, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,404 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87ba365a3e5d028c9193155af96aeb60, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,404 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4197cd4f7ea2f3d6256be6836e9834da, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,405 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=20545c8ee2d66091a91ff9697d1f18ca, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,405 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a50bae1d5cb3d03a86dc103b09a0f68b, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,405 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=215753e705337ae11b1b38ad77b6a5cd, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,405 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e42e4828e881f12c74fbedcdb6b00ea2, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,405 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d63da23e8236e18b416716ec838dd43e, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,405 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e40e25a9323e1d3dad4ec92c578e2f82, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,405 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a2fb6e9d39bf04774472d3b4c70810a, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,405 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1a4ea0dec946dcfe96ad162d5bcf5949, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,405 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06d131e39cb4f5b02db4ec472d438a26, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,406 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0875ddf9cbd035a3345be0684b07b2e6, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,406 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a36d8f99c2aef77f88827585926ad989, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,406 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-02T21:14:47,406 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733174147406 2024-12-02T21:14:47,406 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733174207406 2024-12-02T21:14:47,406 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-12-02T21:14:47,406 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-02T21:14:47,406 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T21:14:47,406 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-02T21:14:47,406 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-02T21:14:47,406 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-02T21:14:47,407 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-02T21:14:47,407 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-02T21:14:47,407 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42045 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:14:47,407 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-02T21:14:47,408 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-02T21:14:47,409 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-02T21:14:47,409 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T21:14:47,409 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T21:14:47,409 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,409 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T21:14:47,409 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T21:14:47,409 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,409 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T21:14:47,410 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-02T21:14:47,410 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-02T21:14:47,410 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=120cf3776e92a15d3276f11911c4042f, REOPEN/MOVE 2024-12-02T21:14:47,411 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true 2024-12-02T21:14:47,411 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:47,411 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:47,411 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T21:14:47,411 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:47,411 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:47,411 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:47,411 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN 2024-12-02T21:14:47,412 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,412 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,412 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,412 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,412 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,412 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,412 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,412 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,412 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN 2024-12-02T21:14:47,412 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-12-02T21:14:47,413 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=65 2024-12-02T21:14:47,413 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=87c3fdb6c570,40413,1733174078061, table=hbase:meta, region=1588230740 2024-12-02T21:14:47,413 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 87c3fdb6c570,40413,1733174078061, state=OPEN 2024-12-02T21:14:47,451 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42045-0x10197e5a0060000, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:47,451 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40413-0x10197e5a0060001, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:47,451 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36585-0x10197e5a0060002, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:47,451 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:47,451 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:47,451 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:47,452 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=06d131e39cb4f5b02db4ec472d438a26, ASSIGN to state=OFFLINE, location=null, table=test, region=06d131e39cb4f5b02db4ec472d438a26 to restore RIT 2024-12-02T21:14:47,452 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733174117452 2024-12-02T21:14:47,452 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-02T21:14:47,452 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:14:47,454 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-02T21:14:47,455 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-02T21:14:47,455 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:47,455 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T21:14:47,455 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:14:47,458 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85ca3a9a09d75c0941c58f5b83584a8e, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,458 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ed86f04f519cc5dccd1b6d41be3da7d, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,458 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0517912ff18e81317bb5474cefaca025, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,458 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff611163b4ab91c50d940651a0065c34, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,458 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=120cf3776e92a15d3276f11911c4042f, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=5 2024-12-02T21:14:47,458 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e87d99b937228303e28b7ea0373ddc70, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,458 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0dfd4169e4c254c51bcc70a472f99ecf, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,458 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9950441eda7548fa66872811a13f5bf0, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,458 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e039115fcc0d0a54e2d03efed7e05de, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,458 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d60c566b39516dd875aaf2bd7def402, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,458 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a524d88be9ac29b416f0a6c04627fa1c, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,458 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0d0be65f6ffd7ebb4c8b2fee37042136, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,458 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=292fd0b4e0f07516cc25c069181e3111, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,459 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87ba365a3e5d028c9193155af96aeb60, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,459 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4197cd4f7ea2f3d6256be6836e9834da, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,459 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=20545c8ee2d66091a91ff9697d1f18ca, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,459 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a50bae1d5cb3d03a86dc103b09a0f68b, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,459 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=215753e705337ae11b1b38ad77b6a5cd, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,459 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e42e4828e881f12c74fbedcdb6b00ea2, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,459 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d63da23e8236e18b416716ec838dd43e, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,459 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e40e25a9323e1d3dad4ec92c578e2f82, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,459 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a2fb6e9d39bf04774472d3b4c70810a, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,459 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1a4ea0dec946dcfe96ad162d5bcf5949, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,459 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06d131e39cb4f5b02db4ec472d438a26, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,459 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0875ddf9cbd035a3345be0684b07b2e6, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,459 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a36d8f99c2aef77f88827585926ad989, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,459 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-02T21:14:47,459 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733174147459 2024-12-02T21:14:47,459 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733174207459 2024-12-02T21:14:47,459 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-12-02T21:14:47,460 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-02T21:14:47,460 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T21:14:47,460 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-02T21:14:47,460 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-02T21:14:47,460 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-02T21:14:47,460 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-02T21:14:47,460 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-02T21:14:47,460 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-02T21:14:47,461 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-02T21:14:47,462 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-02T21:14:47,462 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T21:14:47,462 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T21:14:47,462 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T21:14:47,462 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T21:14:47,462 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,462 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T21:14:47,462 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-02T21:14:47,463 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-02T21:14:47,463 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=120cf3776e92a15d3276f11911c4042f, REOPEN/MOVE 2024-12-02T21:14:47,464 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true 2024-12-02T21:14:47,464 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:47,464 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:47,464 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T21:14:47,464 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:47,464 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:47,464 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:47,464 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,464 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,464 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,464 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,464 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,464 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,464 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,464 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-12-02T21:14:47,465 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=65 2024-12-02T21:14:47,465 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=87c3fdb6c570,40413,1733174078061, table=hbase:meta, region=1588230740 2024-12-02T21:14:47,465 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 87c3fdb6c570,40413,1733174078061, state=OPEN 2024-12-02T21:14:47,476 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36585-0x10197e5a0060002, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:47,476 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42045-0x10197e5a0060000, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:47,476 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40413-0x10197e5a0060001, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:47,476 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:47,476 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:47,476 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:47,477 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733174117477 2024-12-02T21:14:47,477 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-02T21:14:47,477 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:14:47,479 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-02T21:14:47,480 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-02T21:14:47,480 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:47,480 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T21:14:47,480 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:14:47,481 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85ca3a9a09d75c0941c58f5b83584a8e, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ed86f04f519cc5dccd1b6d41be3da7d, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0517912ff18e81317bb5474cefaca025, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff611163b4ab91c50d940651a0065c34, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=120cf3776e92a15d3276f11911c4042f, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=5 2024-12-02T21:14:47,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e87d99b937228303e28b7ea0373ddc70, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0dfd4169e4c254c51bcc70a472f99ecf, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9950441eda7548fa66872811a13f5bf0, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e039115fcc0d0a54e2d03efed7e05de, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d60c566b39516dd875aaf2bd7def402, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a524d88be9ac29b416f0a6c04627fa1c, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0d0be65f6ffd7ebb4c8b2fee37042136, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=292fd0b4e0f07516cc25c069181e3111, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87ba365a3e5d028c9193155af96aeb60, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4197cd4f7ea2f3d6256be6836e9834da, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=20545c8ee2d66091a91ff9697d1f18ca, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,483 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a50bae1d5cb3d03a86dc103b09a0f68b, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,483 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=215753e705337ae11b1b38ad77b6a5cd, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,483 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e42e4828e881f12c74fbedcdb6b00ea2, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,483 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d63da23e8236e18b416716ec838dd43e, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,483 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e40e25a9323e1d3dad4ec92c578e2f82, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,483 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a2fb6e9d39bf04774472d3b4c70810a, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,483 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1a4ea0dec946dcfe96ad162d5bcf5949, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,483 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06d131e39cb4f5b02db4ec472d438a26, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,483 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0875ddf9cbd035a3345be0684b07b2e6, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,483 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a36d8f99c2aef77f88827585926ad989, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,483 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-02T21:14:47,483 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733174147483 2024-12-02T21:14:47,483 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733174207483 2024-12-02T21:14:47,483 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-12-02T21:14:47,483 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-02T21:14:47,483 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T21:14:47,483 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-02T21:14:47,484 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-02T21:14:47,485 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-02T21:14:47,485 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-02T21:14:47,485 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-02T21:14:47,485 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-02T21:14:47,486 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-02T21:14:47,488 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-02T21:14:47,488 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T21:14:47,488 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T21:14:47,488 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T21:14:47,488 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,488 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T21:14:47,488 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-02T21:14:47,489 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-02T21:14:47,489 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=120cf3776e92a15d3276f11911c4042f, REOPEN/MOVE 2024-12-02T21:14:47,490 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true 2024-12-02T21:14:47,490 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:47,490 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:47,490 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T21:14:47,490 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:47,490 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:47,490 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,490 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,490 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,490 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,490 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,490 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.1733174080466, worker=87c3fdb6c570,36585,1733174078182 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,490 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-02T21:14:47,491 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=65 2024-12-02T21:14:47,491 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=87c3fdb6c570,40413,1733174078061, table=hbase:meta, region=1588230740 2024-12-02T21:14:47,492 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 87c3fdb6c570,40413,1733174078061, state=OPEN 2024-12-02T21:14:47,504 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42045-0x10197e5a0060000, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:47,504 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36585-0x10197e5a0060002, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:47,504 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40413-0x10197e5a0060001, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:47,504 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:47,504 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:47,504 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:47,505 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733174117505 2024-12-02T21:14:47,505 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-02T21:14:47,505 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:14:47,506 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-02T21:14:47,507 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-02T21:14:47,508 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:47,508 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T21:14:47,508 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=87c3fdb6c570,40413,1733174078061 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:14:47,511 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85ca3a9a09d75c0941c58f5b83584a8e, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,511 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ed86f04f519cc5dccd1b6d41be3da7d, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,511 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0517912ff18e81317bb5474cefaca025, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,511 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff611163b4ab91c50d940651a0065c34, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,511 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=120cf3776e92a15d3276f11911c4042f, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=5 2024-12-02T21:14:47,511 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e87d99b937228303e28b7ea0373ddc70, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,512 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0dfd4169e4c254c51bcc70a472f99ecf, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,512 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9950441eda7548fa66872811a13f5bf0, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,512 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e039115fcc0d0a54e2d03efed7e05de, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,512 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d60c566b39516dd875aaf2bd7def402, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,512 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a524d88be9ac29b416f0a6c04627fa1c, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,512 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0d0be65f6ffd7ebb4c8b2fee37042136, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,512 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=292fd0b4e0f07516cc25c069181e3111, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,512 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87ba365a3e5d028c9193155af96aeb60, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,512 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4197cd4f7ea2f3d6256be6836e9834da, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,512 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=20545c8ee2d66091a91ff9697d1f18ca, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,513 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a50bae1d5cb3d03a86dc103b09a0f68b, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,513 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=215753e705337ae11b1b38ad77b6a5cd, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,513 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e42e4828e881f12c74fbedcdb6b00ea2, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,513 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d63da23e8236e18b416716ec838dd43e, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,513 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e40e25a9323e1d3dad4ec92c578e2f82, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,513 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a2fb6e9d39bf04774472d3b4c70810a, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,513 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1a4ea0dec946dcfe96ad162d5bcf5949, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,513 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06d131e39cb4f5b02db4ec472d438a26, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,513 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0875ddf9cbd035a3345be0684b07b2e6, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,513 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a36d8f99c2aef77f88827585926ad989, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,513 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-02T21:14:47,513 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733174147513 2024-12-02T21:14:47,513 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733174207513 2024-12-02T21:14:47,513 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-12-02T21:14:47,513 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-02T21:14:47,513 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T21:14:47,513 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-02T21:14:47,514 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-02T21:14:47,514 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-02T21:14:47,514 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-02T21:14:47,514 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-02T21:14:47,514 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-02T21:14:47,515 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-02T21:14:47,516 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-02T21:14:47,516 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T21:14:47,516 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T21:14:47,516 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,517 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-02T21:14:47,517 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-02T21:14:47,517 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-02T21:14:47,517 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=120cf3776e92a15d3276f11911c4042f, REOPEN/MOVE 2024-12-02T21:14:47,518 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true 2024-12-02T21:14:47,518 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:47,518 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:47,518 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T21:14:47,518 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:47,518 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,519 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,519 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,519 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,519 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,519 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-02T21:14:47,522 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=65 2024-12-02T21:14:47,523 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=87c3fdb6c570,40413,1733174078061, table=hbase:meta, region=1588230740 2024-12-02T21:14:47,523 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 87c3fdb6c570,40413,1733174078061, state=OPEN 2024-12-02T21:14:47,535 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42045-0x10197e5a0060000, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:47,535 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40413-0x10197e5a0060001, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:47,535 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36585-0x10197e5a0060002, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:47,535 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:47,535 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:47,536 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:47,537 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733174117537 2024-12-02T21:14:47,537 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-02T21:14:47,537 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:14:47,538 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-02T21:14:47,539 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-02T21:14:47,539 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T21:14:47,539 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T21:14:47,540 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:14:47,542 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85ca3a9a09d75c0941c58f5b83584a8e, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,542 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ed86f04f519cc5dccd1b6d41be3da7d, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,543 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0517912ff18e81317bb5474cefaca025, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,543 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff611163b4ab91c50d940651a0065c34, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,543 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=120cf3776e92a15d3276f11911c4042f, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=5 2024-12-02T21:14:47,543 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e87d99b937228303e28b7ea0373ddc70, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,543 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0dfd4169e4c254c51bcc70a472f99ecf, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,543 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9950441eda7548fa66872811a13f5bf0, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,543 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e039115fcc0d0a54e2d03efed7e05de, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,543 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d60c566b39516dd875aaf2bd7def402, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,543 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a524d88be9ac29b416f0a6c04627fa1c, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,543 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0d0be65f6ffd7ebb4c8b2fee37042136, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,543 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=292fd0b4e0f07516cc25c069181e3111, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,543 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87ba365a3e5d028c9193155af96aeb60, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,544 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4197cd4f7ea2f3d6256be6836e9834da, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,544 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=20545c8ee2d66091a91ff9697d1f18ca, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,544 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a50bae1d5cb3d03a86dc103b09a0f68b, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,544 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=215753e705337ae11b1b38ad77b6a5cd, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,544 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e42e4828e881f12c74fbedcdb6b00ea2, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,544 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d63da23e8236e18b416716ec838dd43e, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,544 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e40e25a9323e1d3dad4ec92c578e2f82, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,544 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a2fb6e9d39bf04774472d3b4c70810a, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,544 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1a4ea0dec946dcfe96ad162d5bcf5949, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,544 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06d131e39cb4f5b02db4ec472d438a26, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,544 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0875ddf9cbd035a3345be0684b07b2e6, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,545 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a36d8f99c2aef77f88827585926ad989, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,545 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-02T21:14:47,545 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733174147545 2024-12-02T21:14:47,545 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733174207545 2024-12-02T21:14:47,545 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-12-02T21:14:47,545 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-02T21:14:47,545 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T21:14:47,545 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-02T21:14:47,545 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-02T21:14:47,545 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-02T21:14:47,545 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-02T21:14:47,545 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-02T21:14:47,546 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-02T21:14:47,547 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-02T21:14:47,548 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-02T21:14:47,548 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T21:14:47,548 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T21:14:47,548 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-02T21:14:47,548 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-02T21:14:47,548 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-02T21:14:47,549 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=120cf3776e92a15d3276f11911c4042f, REOPEN/MOVE 2024-12-02T21:14:47,549 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true 2024-12-02T21:14:47,549 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:47,549 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:47,549 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T21:14:47,549 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,549 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,549 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,549 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,549 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-12-02T21:14:47,550 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=65 2024-12-02T21:14:47,550 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=87c3fdb6c570,40413,1733174078061, table=hbase:meta, region=1588230740 2024-12-02T21:14:47,550 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 87c3fdb6c570,40413,1733174078061, state=OPEN 2024-12-02T21:14:47,560 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42045-0x10197e5a0060000, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:47,560 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40413-0x10197e5a0060001, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:47,560 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36585-0x10197e5a0060002, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:47,561 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:47,561 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:47,561 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:47,562 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733174117562 2024-12-02T21:14:47,562 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-02T21:14:47,562 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:14:47,563 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-02T21:14:47,565 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-02T21:14:47,565 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:47,565 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T21:14:47,565 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:14:47,567 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85ca3a9a09d75c0941c58f5b83584a8e, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,567 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ed86f04f519cc5dccd1b6d41be3da7d, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,567 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0517912ff18e81317bb5474cefaca025, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,567 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff611163b4ab91c50d940651a0065c34, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,567 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=120cf3776e92a15d3276f11911c4042f, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=5 2024-12-02T21:14:47,567 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e87d99b937228303e28b7ea0373ddc70, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,567 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0dfd4169e4c254c51bcc70a472f99ecf, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,567 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9950441eda7548fa66872811a13f5bf0, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,567 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e039115fcc0d0a54e2d03efed7e05de, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,567 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d60c566b39516dd875aaf2bd7def402, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,567 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a524d88be9ac29b416f0a6c04627fa1c, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,567 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0d0be65f6ffd7ebb4c8b2fee37042136, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,567 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=292fd0b4e0f07516cc25c069181e3111, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,568 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87ba365a3e5d028c9193155af96aeb60, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,568 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4197cd4f7ea2f3d6256be6836e9834da, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,568 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=20545c8ee2d66091a91ff9697d1f18ca, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,568 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a50bae1d5cb3d03a86dc103b09a0f68b, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,568 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=215753e705337ae11b1b38ad77b6a5cd, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,568 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e42e4828e881f12c74fbedcdb6b00ea2, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,568 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d63da23e8236e18b416716ec838dd43e, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,568 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e40e25a9323e1d3dad4ec92c578e2f82, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,568 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a2fb6e9d39bf04774472d3b4c70810a, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,568 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1a4ea0dec946dcfe96ad162d5bcf5949, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,568 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06d131e39cb4f5b02db4ec472d438a26, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,568 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0875ddf9cbd035a3345be0684b07b2e6, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,568 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a36d8f99c2aef77f88827585926ad989, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,568 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-02T21:14:47,568 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733174147568 2024-12-02T21:14:47,568 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733174207568 2024-12-02T21:14:47,568 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-12-02T21:14:47,569 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-02T21:14:47,569 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T21:14:47,569 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-02T21:14:47,569 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-02T21:14:47,569 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-02T21:14:47,569 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-02T21:14:47,569 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-02T21:14:47,569 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-02T21:14:47,571 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-02T21:14:47,572 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-02T21:14:47,572 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T21:14:47,572 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-02T21:14:47,572 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-02T21:14:47,573 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-02T21:14:47,573 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=120cf3776e92a15d3276f11911c4042f, REOPEN/MOVE 2024-12-02T21:14:47,573 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true 2024-12-02T21:14:47,573 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:47,573 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:47,574 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,574 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,574 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,574 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-12-02T21:14:47,574 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=65 2024-12-02T21:14:47,574 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=87c3fdb6c570,40413,1733174078061, table=hbase:meta, region=1588230740 2024-12-02T21:14:47,575 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 87c3fdb6c570,40413,1733174078061, state=OPEN 2024-12-02T21:14:47,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36585-0x10197e5a0060002, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:47,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40413-0x10197e5a0060001, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:47,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42045-0x10197e5a0060000, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:47,582 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:47,582 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:47,582 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:47,583 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733174117583 2024-12-02T21:14:47,583 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-02T21:14:47,583 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:14:47,584 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-02T21:14:47,586 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-02T21:14:47,587 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:47,587 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T21:14:47,587 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:14:47,588 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85ca3a9a09d75c0941c58f5b83584a8e, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,588 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ed86f04f519cc5dccd1b6d41be3da7d, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,589 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0517912ff18e81317bb5474cefaca025, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,589 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff611163b4ab91c50d940651a0065c34, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,589 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=120cf3776e92a15d3276f11911c4042f, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=5 2024-12-02T21:14:47,589 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e87d99b937228303e28b7ea0373ddc70, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,589 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0dfd4169e4c254c51bcc70a472f99ecf, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,589 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9950441eda7548fa66872811a13f5bf0, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,589 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e039115fcc0d0a54e2d03efed7e05de, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,589 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d60c566b39516dd875aaf2bd7def402, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,589 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a524d88be9ac29b416f0a6c04627fa1c, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,589 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0d0be65f6ffd7ebb4c8b2fee37042136, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,589 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=292fd0b4e0f07516cc25c069181e3111, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,589 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87ba365a3e5d028c9193155af96aeb60, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,589 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4197cd4f7ea2f3d6256be6836e9834da, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,590 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=20545c8ee2d66091a91ff9697d1f18ca, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,590 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a50bae1d5cb3d03a86dc103b09a0f68b, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,590 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=215753e705337ae11b1b38ad77b6a5cd, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,590 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e42e4828e881f12c74fbedcdb6b00ea2, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,590 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d63da23e8236e18b416716ec838dd43e, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,590 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e40e25a9323e1d3dad4ec92c578e2f82, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,590 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a2fb6e9d39bf04774472d3b4c70810a, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,590 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1a4ea0dec946dcfe96ad162d5bcf5949, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,590 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06d131e39cb4f5b02db4ec472d438a26, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,590 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0875ddf9cbd035a3345be0684b07b2e6, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,590 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a36d8f99c2aef77f88827585926ad989, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,590 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-02T21:14:47,590 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733174147590 2024-12-02T21:14:47,590 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733174207590 2024-12-02T21:14:47,590 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-12-02T21:14:47,590 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-02T21:14:47,591 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T21:14:47,591 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-02T21:14:47,591 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-02T21:14:47,591 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-02T21:14:47,591 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-02T21:14:47,591 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-02T21:14:47,591 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-02T21:14:47,592 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-02T21:14:47,593 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-02T21:14:47,593 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-02T21:14:47,594 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-02T21:14:47,594 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-02T21:14:47,594 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=120cf3776e92a15d3276f11911c4042f, REOPEN/MOVE 2024-12-02T21:14:47,594 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true 2024-12-02T21:14:47,594 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:47,594 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,594 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 87c3fdb6c570%2C33459%2C1733174078298.meta.1733174080862.meta, worker=87c3fdb6c570,40413,1733174078061 didn't hold the lock before restarting, skip acquiring lock. 2024-12-02T21:14:47,595 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-12-02T21:14:47,595 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=65 2024-12-02T21:14:47,595 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=87c3fdb6c570,40413,1733174078061, table=hbase:meta, region=1588230740 2024-12-02T21:14:47,595 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 87c3fdb6c570,40413,1733174078061, state=OPEN 2024-12-02T21:14:47,602 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36585-0x10197e5a0060002, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:47,602 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40413-0x10197e5a0060001, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:47,602 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42045-0x10197e5a0060000, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:47,603 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:47,603 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:47,603 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:47,603 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733174117603 2024-12-02T21:14:47,603 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-02T21:14:47,603 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:14:47,605 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-02T21:14:47,608 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-02T21:14:47,608 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true 2024-12-02T21:14:47,608 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85ca3a9a09d75c0941c58f5b83584a8e, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,608 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T21:14:47,608 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ed86f04f519cc5dccd1b6d41be3da7d, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,608 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0517912ff18e81317bb5474cefaca025, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,608 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1604) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:14:47,608 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff611163b4ab91c50d940651a0065c34, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,608 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=120cf3776e92a15d3276f11911c4042f, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=5 2024-12-02T21:14:47,609 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e87d99b937228303e28b7ea0373ddc70, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,609 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0dfd4169e4c254c51bcc70a472f99ecf, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,609 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9950441eda7548fa66872811a13f5bf0, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,609 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e039115fcc0d0a54e2d03efed7e05de, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,609 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d60c566b39516dd875aaf2bd7def402, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,609 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a524d88be9ac29b416f0a6c04627fa1c, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,609 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0d0be65f6ffd7ebb4c8b2fee37042136, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,609 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=292fd0b4e0f07516cc25c069181e3111, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,609 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87ba365a3e5d028c9193155af96aeb60, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,609 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4197cd4f7ea2f3d6256be6836e9834da, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,609 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=20545c8ee2d66091a91ff9697d1f18ca, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,609 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a50bae1d5cb3d03a86dc103b09a0f68b, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,609 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=215753e705337ae11b1b38ad77b6a5cd, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,609 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e42e4828e881f12c74fbedcdb6b00ea2, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,609 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d63da23e8236e18b416716ec838dd43e, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,609 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e40e25a9323e1d3dad4ec92c578e2f82, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,609 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a2fb6e9d39bf04774472d3b4c70810a, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,610 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1a4ea0dec946dcfe96ad162d5bcf5949, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,610 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06d131e39cb4f5b02db4ec472d438a26, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,610 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0875ddf9cbd035a3345be0684b07b2e6, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,610 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a36d8f99c2aef77f88827585926ad989, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,610 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-02T21:14:47,610 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733174147610 2024-12-02T21:14:47,610 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733174207610 2024-12-02T21:14:47,610 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-12-02T21:14:47,610 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-02T21:14:47,610 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T21:14:47,610 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-02T21:14:47,610 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-02T21:14:47,610 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-02T21:14:47,610 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-02T21:14:47,610 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-02T21:14:47,611 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-02T21:14:47,611 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-02T21:14:47,612 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42045 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:14:47,612 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-02T21:14:47,613 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-02T21:14:47,613 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42045 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:14:47,613 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-02T21:14:47,613 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=120cf3776e92a15d3276f11911c4042f, REOPEN/MOVE 2024-12-02T21:14:47,614 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true 2024-12-02T21:14:47,614 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-02T21:14:47,614 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-12-02T21:14:47,614 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=65 2024-12-02T21:14:47,615 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=87c3fdb6c570,40413,1733174078061, table=hbase:meta, region=1588230740 2024-12-02T21:14:47,615 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 87c3fdb6c570,40413,1733174078061, state=OPEN 2024-12-02T21:14:47,623 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36585-0x10197e5a0060002, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:47,623 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42045-0x10197e5a0060000, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:47,623 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40413-0x10197e5a0060001, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T21:14:47,624 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:47,624 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:47,624 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T21:14:47,625 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733174117625 2024-12-02T21:14:47,625 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-02T21:14:47,625 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T21:14:47,626 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-02T21:14:47,627 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=60, state=ROLLEDBACK, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=73, state=RUNNABLE, hasLock=true; OpenRegionProcedure d63da23e8236e18b416716ec838dd43e, server=87c3fdb6c570,36585,1733174078182:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 87c3fdb6c570,33459,1733174078298, splitWal=true, meta=true exec-time=2.8470 sec 2024-12-02T21:14:47,629 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85ca3a9a09d75c0941c58f5b83584a8e, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,629 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ed86f04f519cc5dccd1b6d41be3da7d, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,629 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0517912ff18e81317bb5474cefaca025, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,629 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff611163b4ab91c50d940651a0065c34, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,629 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=120cf3776e92a15d3276f11911c4042f, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=5 2024-12-02T21:14:47,630 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e87d99b937228303e28b7ea0373ddc70, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,630 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0dfd4169e4c254c51bcc70a472f99ecf, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,630 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9950441eda7548fa66872811a13f5bf0, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,630 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e039115fcc0d0a54e2d03efed7e05de, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,630 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d60c566b39516dd875aaf2bd7def402, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,630 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a524d88be9ac29b416f0a6c04627fa1c, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,630 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0d0be65f6ffd7ebb4c8b2fee37042136, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,630 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=292fd0b4e0f07516cc25c069181e3111, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,630 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87ba365a3e5d028c9193155af96aeb60, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,630 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4197cd4f7ea2f3d6256be6836e9834da, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,630 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=20545c8ee2d66091a91ff9697d1f18ca, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,630 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a50bae1d5cb3d03a86dc103b09a0f68b, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,630 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=215753e705337ae11b1b38ad77b6a5cd, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,630 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e42e4828e881f12c74fbedcdb6b00ea2, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,630 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d63da23e8236e18b416716ec838dd43e, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,630 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e40e25a9323e1d3dad4ec92c578e2f82, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,630 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a2fb6e9d39bf04774472d3b4c70810a, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,631 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1a4ea0dec946dcfe96ad162d5bcf5949, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,631 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06d131e39cb4f5b02db4ec472d438a26, regionState=OPENING, lastHost=87c3fdb6c570,33459,1733174078298, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,631 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0875ddf9cbd035a3345be0684b07b2e6, regionState=OPEN, lastHost=87c3fdb6c570,36585,1733174078182, regionLocation=87c3fdb6c570,36585,1733174078182, openSeqNum=2 2024-12-02T21:14:47,631 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a36d8f99c2aef77f88827585926ad989, regionState=OPEN, lastHost=87c3fdb6c570,40413,1733174078061, regionLocation=87c3fdb6c570,40413,1733174078061, openSeqNum=2 2024-12-02T21:14:47,631 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-02T21:14:47,631 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733174147631 2024-12-02T21:14:47,631 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733174207631 2024-12-02T21:14:47,631 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-12-02T21:14:47,644 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.assignment.TestRollbackSCP#testFailAndRollback Thread=346 (was 380), OpenFileDescriptor=618 (was 641), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=779 (was 821), ProcessCount=11 (was 11), AvailableMemoryMB=2810 (was 2880) 2024-12-02T21:14:47,644 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-02T21:14:47,644 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T21:14:47,644 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T21:14:47,644 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:14:47,645 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:14:47,645 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T21:14:47,645 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-02T21:14:47,645 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=189905736, stopped=false 2024-12-02T21:14:47,646 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=87c3fdb6c570,42045,1733174077295 2024-12-02T21:14:47,649 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-02T21:14:47,650 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-02T21:14:47,651 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test 2024-12-02T21:14:47,651 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test Metrics about Tables on a single HBase RegionServer 2024-12-02T21:14:47,651 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-02T21:14:47,651 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-02T21:14:47,651 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-02T21:14:47,651 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-02T21:14:47,655 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40413-0x10197e5a0060001, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-02T21:14:47,655 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36585-0x10197e5a0060002, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-02T21:14:47,655 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40413-0x10197e5a0060001, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T21:14:47,655 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42045-0x10197e5a0060000, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-02T21:14:47,655 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36585-0x10197e5a0060002, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T21:14:47,655 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42045-0x10197e5a0060000, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T21:14:47,655 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T21:14:47,655 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T21:14:47,656 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T21:14:47,656 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40413-0x10197e5a0060001, quorum=127.0.0.1:63400, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T21:14:47,656 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42045-0x10197e5a0060000, quorum=127.0.0.1:63400, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T21:14:47,656 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:14:47,656 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36585-0x10197e5a0060002, quorum=127.0.0.1:63400, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T21:14:47,656 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '87c3fdb6c570,40413,1733174078061' ***** 2024-12-02T21:14:47,656 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-02T21:14:47,656 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '87c3fdb6c570,36585,1733174078182' ***** 2024-12-02T21:14:47,656 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-02T21:14:47,656 INFO [RS:0;87c3fdb6c570:40413 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-02T21:14:47,656 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-02T21:14:47,656 INFO [RS:0;87c3fdb6c570:40413 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-02T21:14:47,656 INFO [RS:0;87c3fdb6c570:40413 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-02T21:14:47,657 INFO [RS:1;87c3fdb6c570:36585 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-02T21:14:47,657 INFO [RS:0;87c3fdb6c570:40413 {}] regionserver.HRegionServer(3091): Received CLOSE for 9950441eda7548fa66872811a13f5bf0 2024-12-02T21:14:47,657 INFO [RS:1;87c3fdb6c570:36585 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-02T21:14:47,657 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-02T21:14:47,657 INFO [RS:1;87c3fdb6c570:36585 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-02T21:14:47,657 INFO [RS:1;87c3fdb6c570:36585 {}] regionserver.HRegionServer(3091): Received CLOSE for e87d99b937228303e28b7ea0373ddc70 2024-12-02T21:14:47,657 INFO [RS:0;87c3fdb6c570:40413 {}] regionserver.HRegionServer(3091): Received CLOSE for 5ed86f04f519cc5dccd1b6d41be3da7d 2024-12-02T21:14:47,657 INFO [RS:0;87c3fdb6c570:40413 {}] regionserver.HRegionServer(3091): Received CLOSE for a524d88be9ac29b416f0a6c04627fa1c 2024-12-02T21:14:47,657 INFO [RS:1;87c3fdb6c570:36585 {}] regionserver.HRegionServer(3091): Received CLOSE for e40e25a9323e1d3dad4ec92c578e2f82 2024-12-02T21:14:47,657 INFO [RS:0;87c3fdb6c570:40413 {}] regionserver.HRegionServer(3091): Received CLOSE for ff611163b4ab91c50d940651a0065c34 2024-12-02T21:14:47,657 INFO [RS:0;87c3fdb6c570:40413 {}] regionserver.HRegionServer(3091): Received CLOSE for e42e4828e881f12c74fbedcdb6b00ea2 2024-12-02T21:14:47,657 INFO [RS:1;87c3fdb6c570:36585 {}] regionserver.HRegionServer(3091): Received CLOSE for 215753e705337ae11b1b38ad77b6a5cd 2024-12-02T21:14:47,657 INFO [RS:0;87c3fdb6c570:40413 {}] regionserver.HRegionServer(3091): Received CLOSE for a36d8f99c2aef77f88827585926ad989 2024-12-02T21:14:47,657 INFO [RS:1;87c3fdb6c570:36585 {}] regionserver.HRegionServer(3091): Received CLOSE for 85ca3a9a09d75c0941c58f5b83584a8e 2024-12-02T21:14:47,657 INFO [RS:0;87c3fdb6c570:40413 {}] regionserver.HRegionServer(3091): Received CLOSE for 292fd0b4e0f07516cc25c069181e3111 2024-12-02T21:14:47,658 INFO [RS:0;87c3fdb6c570:40413 {}] regionserver.HRegionServer(3091): Received CLOSE for 6a2fb6e9d39bf04774472d3b4c70810a 2024-12-02T21:14:47,658 INFO [RS:1;87c3fdb6c570:36585 {}] regionserver.HRegionServer(3091): Received CLOSE for 4197cd4f7ea2f3d6256be6836e9834da 2024-12-02T21:14:47,658 INFO [RS:0;87c3fdb6c570:40413 {}] regionserver.HRegionServer(3091): Received CLOSE for 20545c8ee2d66091a91ff9697d1f18ca 2024-12-02T21:14:47,658 INFO [RS:1;87c3fdb6c570:36585 {}] regionserver.HRegionServer(3091): Received CLOSE for d63da23e8236e18b416716ec838dd43e 2024-12-02T21:14:47,658 INFO [RS:1;87c3fdb6c570:36585 {}] regionserver.HRegionServer(3091): Received CLOSE for 8e039115fcc0d0a54e2d03efed7e05de 2024-12-02T21:14:47,658 INFO [RS:0;87c3fdb6c570:40413 {}] regionserver.HRegionServer(959): stopping server 87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:47,658 INFO [RS:0;87c3fdb6c570:40413 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-02T21:14:47,658 INFO [RS:1;87c3fdb6c570:36585 {}] regionserver.HRegionServer(3091): Received CLOSE for 0875ddf9cbd035a3345be0684b07b2e6 2024-12-02T21:14:47,658 INFO [RS:0;87c3fdb6c570:40413 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;87c3fdb6c570:40413. 2024-12-02T21:14:47,658 INFO [RS:1;87c3fdb6c570:36585 {}] regionserver.HRegionServer(3091): Received CLOSE for 87ba365a3e5d028c9193155af96aeb60 2024-12-02T21:14:47,658 INFO [RS:1;87c3fdb6c570:36585 {}] regionserver.HRegionServer(3091): Received CLOSE for 06d131e39cb4f5b02db4ec472d438a26 2024-12-02T21:14:47,658 DEBUG [RS:0;87c3fdb6c570:40413 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T21:14:47,658 DEBUG [RS:0;87c3fdb6c570:40413 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:14:47,658 INFO [RS:1;87c3fdb6c570:36585 {}] regionserver.HRegionServer(3091): Received CLOSE for 0dfd4169e4c254c51bcc70a472f99ecf 2024-12-02T21:14:47,658 INFO [RS:1;87c3fdb6c570:36585 {}] regionserver.HRegionServer(3091): Received CLOSE for 5d60c566b39516dd875aaf2bd7def402 2024-12-02T21:14:47,658 INFO [RS:1;87c3fdb6c570:36585 {}] regionserver.HRegionServer(3091): Received CLOSE for 120cf3776e92a15d3276f11911c4042f 2024-12-02T21:14:47,658 INFO [RS:1;87c3fdb6c570:36585 {}] regionserver.HRegionServer(959): stopping server 87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:47,658 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e87d99b937228303e28b7ea0373ddc70, disabling compactions & flushes 2024-12-02T21:14:47,658 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9950441eda7548fa66872811a13f5bf0, disabling compactions & flushes 2024-12-02T21:14:47,658 INFO [RS:0;87c3fdb6c570:40413 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-02T21:14:47,658 INFO [RS:1;87c3fdb6c570:36585 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-02T21:14:47,658 INFO [RS:0;87c3fdb6c570:40413 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-02T21:14:47,658 INFO [RS:1;87c3fdb6c570:36585 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;87c3fdb6c570:36585. 2024-12-02T21:14:47,658 INFO [RS:0;87c3fdb6c570:40413 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-02T21:14:47,658 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,hhh,1733174081291.9950441eda7548fa66872811a13f5bf0. 2024-12-02T21:14:47,658 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,fff,1733174081291.e87d99b937228303e28b7ea0373ddc70. 2024-12-02T21:14:47,658 INFO [RS:0;87c3fdb6c570:40413 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-02T21:14:47,658 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1733174081291.9950441eda7548fa66872811a13f5bf0. 2024-12-02T21:14:47,658 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1733174081291.e87d99b937228303e28b7ea0373ddc70. 2024-12-02T21:14:47,658 DEBUG [RS:1;87c3fdb6c570:36585 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T21:14:47,659 DEBUG [RS:1;87c3fdb6c570:36585 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:14:47,659 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,fff,1733174081291.e87d99b937228303e28b7ea0373ddc70. after waiting 0 ms 2024-12-02T21:14:47,659 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1733174081291.9950441eda7548fa66872811a13f5bf0. after waiting 0 ms 2024-12-02T21:14:47,659 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,fff,1733174081291.e87d99b937228303e28b7ea0373ddc70. 2024-12-02T21:14:47,659 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1733174081291.9950441eda7548fa66872811a13f5bf0. 2024-12-02T21:14:47,659 INFO [RS:1;87c3fdb6c570:36585 {}] regionserver.HRegionServer(1321): Waiting on 13 regions to close 2024-12-02T21:14:47,659 DEBUG [RS:1;87c3fdb6c570:36585 {}] regionserver.HRegionServer(1351): Waiting on 06d131e39cb4f5b02db4ec472d438a26, 0875ddf9cbd035a3345be0684b07b2e6, 0dfd4169e4c254c51bcc70a472f99ecf, 120cf3776e92a15d3276f11911c4042f, 215753e705337ae11b1b38ad77b6a5cd, 4197cd4f7ea2f3d6256be6836e9834da, 5d60c566b39516dd875aaf2bd7def402, 85ca3a9a09d75c0941c58f5b83584a8e, 87ba365a3e5d028c9193155af96aeb60, 8e039115fcc0d0a54e2d03efed7e05de, d63da23e8236e18b416716ec838dd43e, e40e25a9323e1d3dad4ec92c578e2f82, e87d99b937228303e28b7ea0373ddc70 2024-12-02T21:14:47,659 INFO [RS:0;87c3fdb6c570:40413 {}] regionserver.HRegionServer(1321): Waiting on 10 regions to close 2024-12-02T21:14:47,659 DEBUG [RS:0;87c3fdb6c570:40413 {}] regionserver.HRegionServer(1351): Waiting on 0d0be65f6ffd7ebb4c8b2fee37042136, 1588230740, 20545c8ee2d66091a91ff9697d1f18ca, 292fd0b4e0f07516cc25c069181e3111, 5ed86f04f519cc5dccd1b6d41be3da7d, 6a2fb6e9d39bf04774472d3b4c70810a, 9950441eda7548fa66872811a13f5bf0, a36d8f99c2aef77f88827585926ad989, a524d88be9ac29b416f0a6c04627fa1c, e42e4828e881f12c74fbedcdb6b00ea2, ff611163b4ab91c50d940651a0065c34 2024-12-02T21:14:47,660 DEBUG [RS_CLOSE_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-02T21:14:47,660 INFO [RS_CLOSE_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-02T21:14:47,660 DEBUG [RS_CLOSE_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-02T21:14:47,660 DEBUG [RS_CLOSE_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-02T21:14:47,660 DEBUG [RS_CLOSE_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-02T21:14:47,660 INFO [RS_CLOSE_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.66 KB heapSize=5.44 KB 2024-12-02T21:14:47,665 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/e87d99b937228303e28b7ea0373ddc70/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T21:14:47,666 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,fff,1733174081291.e87d99b937228303e28b7ea0373ddc70. 2024-12-02T21:14:47,666 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e87d99b937228303e28b7ea0373ddc70: Waiting for close lock at 1733174087657Running coprocessor pre-close hooks at 1733174087658 (+1 ms)Disabling compacts and flushes for region at 1733174087658Disabling writes for close at 1733174087659 (+1 ms)Writing region close event to WAL at 1733174087660 (+1 ms)Running coprocessor post-close hooks at 1733174087666 (+6 ms)Closed at 1733174087666 2024-12-02T21:14:47,666 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,fff,1733174081291.e87d99b937228303e28b7ea0373ddc70. 2024-12-02T21:14:47,666 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e40e25a9323e1d3dad4ec92c578e2f82, disabling compactions & flushes 2024-12-02T21:14:47,666 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,uuu,1733174081291.e40e25a9323e1d3dad4ec92c578e2f82. 2024-12-02T21:14:47,666 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1733174081291.e40e25a9323e1d3dad4ec92c578e2f82. 2024-12-02T21:14:47,667 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1733174081291.e40e25a9323e1d3dad4ec92c578e2f82. after waiting 0 ms 2024-12-02T21:14:47,667 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1733174081291.e40e25a9323e1d3dad4ec92c578e2f82. 2024-12-02T21:14:47,668 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/9950441eda7548fa66872811a13f5bf0/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T21:14:47,669 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,hhh,1733174081291.9950441eda7548fa66872811a13f5bf0. 2024-12-02T21:14:47,669 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9950441eda7548fa66872811a13f5bf0: Waiting for close lock at 1733174087657Running coprocessor pre-close hooks at 1733174087658 (+1 ms)Disabling compacts and flushes for region at 1733174087658Disabling writes for close at 1733174087659 (+1 ms)Writing region close event to WAL at 1733174087660 (+1 ms)Running coprocessor post-close hooks at 1733174087669 (+9 ms)Closed at 1733174087669 2024-12-02T21:14:47,669 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,hhh,1733174081291.9950441eda7548fa66872811a13f5bf0. 2024-12-02T21:14:47,670 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5ed86f04f519cc5dccd1b6d41be3da7d, disabling compactions & flushes 2024-12-02T21:14:47,670 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,bbb,1733174081291.5ed86f04f519cc5dccd1b6d41be3da7d. 2024-12-02T21:14:47,670 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1733174081291.5ed86f04f519cc5dccd1b6d41be3da7d. 2024-12-02T21:14:47,670 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1733174081291.5ed86f04f519cc5dccd1b6d41be3da7d. after waiting 0 ms 2024-12-02T21:14:47,670 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1733174081291.5ed86f04f519cc5dccd1b6d41be3da7d. 2024-12-02T21:14:47,671 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/e40e25a9323e1d3dad4ec92c578e2f82/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T21:14:47,672 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,uuu,1733174081291.e40e25a9323e1d3dad4ec92c578e2f82. 2024-12-02T21:14:47,672 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e40e25a9323e1d3dad4ec92c578e2f82: Waiting for close lock at 1733174087666Running coprocessor pre-close hooks at 1733174087666Disabling compacts and flushes for region at 1733174087666Disabling writes for close at 1733174087667 (+1 ms)Writing region close event to WAL at 1733174087668 (+1 ms)Running coprocessor post-close hooks at 1733174087672 (+4 ms)Closed at 1733174087672 2024-12-02T21:14:47,672 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,uuu,1733174081291.e40e25a9323e1d3dad4ec92c578e2f82. 2024-12-02T21:14:47,672 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 215753e705337ae11b1b38ad77b6a5cd, disabling compactions & flushes 2024-12-02T21:14:47,672 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,rrr,1733174081291.215753e705337ae11b1b38ad77b6a5cd. 2024-12-02T21:14:47,673 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1733174081291.215753e705337ae11b1b38ad77b6a5cd. 2024-12-02T21:14:47,673 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1733174081291.215753e705337ae11b1b38ad77b6a5cd. after waiting 0 ms 2024-12-02T21:14:47,673 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1733174081291.215753e705337ae11b1b38ad77b6a5cd. 2024-12-02T21:14:47,676 DEBUG [RS_CLOSE_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/hbase/meta/1588230740/.tmp/info/027d1be57ffd4a7d94d41f5e364beb1c is 138, key is test,ccc,1733174081291.0517912ff18e81317bb5474cefaca025./info:regioninfo/1733174086525/Put/seqid=0 2024-12-02T21:14:47,678 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/5ed86f04f519cc5dccd1b6d41be3da7d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T21:14:47,679 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,bbb,1733174081291.5ed86f04f519cc5dccd1b6d41be3da7d. 2024-12-02T21:14:47,679 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5ed86f04f519cc5dccd1b6d41be3da7d: Waiting for close lock at 1733174087670Running coprocessor pre-close hooks at 1733174087670Disabling compacts and flushes for region at 1733174087670Disabling writes for close at 1733174087670Writing region close event to WAL at 1733174087672 (+2 ms)Running coprocessor post-close hooks at 1733174087679 (+7 ms)Closed at 1733174087679 2024-12-02T21:14:47,679 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,bbb,1733174081291.5ed86f04f519cc5dccd1b6d41be3da7d. 2024-12-02T21:14:47,679 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a524d88be9ac29b416f0a6c04627fa1c, disabling compactions & flushes 2024-12-02T21:14:47,679 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,kkk,1733174081291.a524d88be9ac29b416f0a6c04627fa1c. 2024-12-02T21:14:47,679 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1733174081291.a524d88be9ac29b416f0a6c04627fa1c. 2024-12-02T21:14:47,679 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1733174081291.a524d88be9ac29b416f0a6c04627fa1c. after waiting 0 ms 2024-12-02T21:14:47,679 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1733174081291.a524d88be9ac29b416f0a6c04627fa1c. 2024-12-02T21:14:47,685 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/215753e705337ae11b1b38ad77b6a5cd/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T21:14:47,686 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,rrr,1733174081291.215753e705337ae11b1b38ad77b6a5cd. 2024-12-02T21:14:47,686 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 215753e705337ae11b1b38ad77b6a5cd: Waiting for close lock at 1733174087672Running coprocessor pre-close hooks at 1733174087672Disabling compacts and flushes for region at 1733174087672Disabling writes for close at 1733174087673 (+1 ms)Writing region close event to WAL at 1733174087673Running coprocessor post-close hooks at 1733174087686 (+13 ms)Closed at 1733174087686 2024-12-02T21:14:47,687 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,rrr,1733174081291.215753e705337ae11b1b38ad77b6a5cd. 2024-12-02T21:14:47,687 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 85ca3a9a09d75c0941c58f5b83584a8e, disabling compactions & flushes 2024-12-02T21:14:47,687 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,,1733174081291.85ca3a9a09d75c0941c58f5b83584a8e. 2024-12-02T21:14:47,687 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1733174081291.85ca3a9a09d75c0941c58f5b83584a8e. 2024-12-02T21:14:47,687 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,,1733174081291.85ca3a9a09d75c0941c58f5b83584a8e. after waiting 0 ms 2024-12-02T21:14:47,687 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,,1733174081291.85ca3a9a09d75c0941c58f5b83584a8e. 2024-12-02T21:14:47,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36203 is added to blk_1073741869_1045 (size=8195) 2024-12-02T21:14:47,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741869_1045 (size=8195) 2024-12-02T21:14:47,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42827 is added to blk_1073741869_1045 (size=8195) 2024-12-02T21:14:47,695 INFO [RS_CLOSE_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.66 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/hbase/meta/1588230740/.tmp/info/027d1be57ffd4a7d94d41f5e364beb1c 2024-12-02T21:14:47,701 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/85ca3a9a09d75c0941c58f5b83584a8e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T21:14:47,701 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/a524d88be9ac29b416f0a6c04627fa1c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T21:14:47,702 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,,1733174081291.85ca3a9a09d75c0941c58f5b83584a8e. 2024-12-02T21:14:47,702 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 85ca3a9a09d75c0941c58f5b83584a8e: Waiting for close lock at 1733174087687Running coprocessor pre-close hooks at 1733174087687Disabling compacts and flushes for region at 1733174087687Disabling writes for close at 1733174087687Writing region close event to WAL at 1733174087687Running coprocessor post-close hooks at 1733174087702 (+15 ms)Closed at 1733174087702 2024-12-02T21:14:47,702 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,,1733174081291.85ca3a9a09d75c0941c58f5b83584a8e. 2024-12-02T21:14:47,702 INFO [regionserver/87c3fdb6c570:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-02T21:14:47,702 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,kkk,1733174081291.a524d88be9ac29b416f0a6c04627fa1c. 2024-12-02T21:14:47,702 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4197cd4f7ea2f3d6256be6836e9834da, disabling compactions & flushes 2024-12-02T21:14:47,702 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a524d88be9ac29b416f0a6c04627fa1c: Waiting for close lock at 1733174087679Running coprocessor pre-close hooks at 1733174087679Disabling compacts and flushes for region at 1733174087679Disabling writes for close at 1733174087679Writing region close event to WAL at 1733174087685 (+6 ms)Running coprocessor post-close hooks at 1733174087702 (+17 ms)Closed at 1733174087702 2024-12-02T21:14:47,702 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ooo,1733174081291.4197cd4f7ea2f3d6256be6836e9834da. 2024-12-02T21:14:47,702 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1733174081291.4197cd4f7ea2f3d6256be6836e9834da. 2024-12-02T21:14:47,702 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,kkk,1733174081291.a524d88be9ac29b416f0a6c04627fa1c. 2024-12-02T21:14:47,702 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1733174081291.4197cd4f7ea2f3d6256be6836e9834da. after waiting 0 ms 2024-12-02T21:14:47,702 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1733174081291.4197cd4f7ea2f3d6256be6836e9834da. 2024-12-02T21:14:47,702 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ff611163b4ab91c50d940651a0065c34, disabling compactions & flushes 2024-12-02T21:14:47,703 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ddd,1733174081291.ff611163b4ab91c50d940651a0065c34. 2024-12-02T21:14:47,703 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1733174081291.ff611163b4ab91c50d940651a0065c34. 2024-12-02T21:14:47,703 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1733174081291.ff611163b4ab91c50d940651a0065c34. after waiting 0 ms 2024-12-02T21:14:47,703 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1733174081291.ff611163b4ab91c50d940651a0065c34. 2024-12-02T21:14:47,704 DEBUG [RS_CLOSE_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/hbase/meta/1588230740/.tmp/info/027d1be57ffd4a7d94d41f5e364beb1c as hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/hbase/meta/1588230740/info/027d1be57ffd4a7d94d41f5e364beb1c 2024-12-02T21:14:47,707 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/ff611163b4ab91c50d940651a0065c34/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T21:14:47,708 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/4197cd4f7ea2f3d6256be6836e9834da/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T21:14:47,708 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ddd,1733174081291.ff611163b4ab91c50d940651a0065c34. 2024-12-02T21:14:47,708 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ff611163b4ab91c50d940651a0065c34: Waiting for close lock at 1733174087702Running coprocessor pre-close hooks at 1733174087702Disabling compacts and flushes for region at 1733174087702Disabling writes for close at 1733174087703 (+1 ms)Writing region close event to WAL at 1733174087704 (+1 ms)Running coprocessor post-close hooks at 1733174087708 (+4 ms)Closed at 1733174087708 2024-12-02T21:14:47,708 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ddd,1733174081291.ff611163b4ab91c50d940651a0065c34. 2024-12-02T21:14:47,708 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e42e4828e881f12c74fbedcdb6b00ea2, disabling compactions & flushes 2024-12-02T21:14:47,709 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,sss,1733174081291.e42e4828e881f12c74fbedcdb6b00ea2. 2024-12-02T21:14:47,709 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1733174081291.e42e4828e881f12c74fbedcdb6b00ea2. 2024-12-02T21:14:47,709 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,sss,1733174081291.e42e4828e881f12c74fbedcdb6b00ea2. after waiting 0 ms 2024-12-02T21:14:47,709 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,sss,1733174081291.e42e4828e881f12c74fbedcdb6b00ea2. 2024-12-02T21:14:47,709 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ooo,1733174081291.4197cd4f7ea2f3d6256be6836e9834da. 2024-12-02T21:14:47,709 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4197cd4f7ea2f3d6256be6836e9834da: Waiting for close lock at 1733174087702Running coprocessor pre-close hooks at 1733174087702Disabling compacts and flushes for region at 1733174087702Disabling writes for close at 1733174087702Writing region close event to WAL at 1733174087703 (+1 ms)Running coprocessor post-close hooks at 1733174087709 (+6 ms)Closed at 1733174087709 2024-12-02T21:14:47,709 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ooo,1733174081291.4197cd4f7ea2f3d6256be6836e9834da. 2024-12-02T21:14:47,709 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d63da23e8236e18b416716ec838dd43e, disabling compactions & flushes 2024-12-02T21:14:47,709 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ttt,1733174081291.d63da23e8236e18b416716ec838dd43e. 2024-12-02T21:14:47,710 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1733174081291.d63da23e8236e18b416716ec838dd43e. 2024-12-02T21:14:47,710 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1733174081291.d63da23e8236e18b416716ec838dd43e. after waiting 0 ms 2024-12-02T21:14:47,710 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1733174081291.d63da23e8236e18b416716ec838dd43e. 2024-12-02T21:14:47,713 INFO [regionserver/87c3fdb6c570:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-02T21:14:47,715 INFO [RS_CLOSE_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/hbase/meta/1588230740/info/027d1be57ffd4a7d94d41f5e364beb1c, entries=24, sequenceid=75, filesize=8.0 K 2024-12-02T21:14:47,715 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/e42e4828e881f12c74fbedcdb6b00ea2/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T21:14:47,716 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/d63da23e8236e18b416716ec838dd43e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T21:14:47,716 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,sss,1733174081291.e42e4828e881f12c74fbedcdb6b00ea2. 2024-12-02T21:14:47,716 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e42e4828e881f12c74fbedcdb6b00ea2: Waiting for close lock at 1733174087708Running coprocessor pre-close hooks at 1733174087708Disabling compacts and flushes for region at 1733174087708Disabling writes for close at 1733174087709 (+1 ms)Writing region close event to WAL at 1733174087710 (+1 ms)Running coprocessor post-close hooks at 1733174087716 (+6 ms)Closed at 1733174087716 2024-12-02T21:14:47,716 INFO [RS_CLOSE_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.66 KB/2720, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 56ms, sequenceid=75, compaction requested=false 2024-12-02T21:14:47,716 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,sss,1733174081291.e42e4828e881f12c74fbedcdb6b00ea2. 2024-12-02T21:14:47,716 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a36d8f99c2aef77f88827585926ad989, disabling compactions & flushes 2024-12-02T21:14:47,716 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ttt,1733174081291.d63da23e8236e18b416716ec838dd43e. 2024-12-02T21:14:47,716 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,zzz,1733174081291.a36d8f99c2aef77f88827585926ad989. 2024-12-02T21:14:47,717 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d63da23e8236e18b416716ec838dd43e: Waiting for close lock at 1733174087709Running coprocessor pre-close hooks at 1733174087709Disabling compacts and flushes for region at 1733174087709Disabling writes for close at 1733174087710 (+1 ms)Writing region close event to WAL at 1733174087711 (+1 ms)Running coprocessor post-close hooks at 1733174087716 (+5 ms)Closed at 1733174087716 2024-12-02T21:14:47,717 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1733174081291.a36d8f99c2aef77f88827585926ad989. 2024-12-02T21:14:47,717 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1733174081291.a36d8f99c2aef77f88827585926ad989. after waiting 0 ms 2024-12-02T21:14:47,717 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1733174081291.a36d8f99c2aef77f88827585926ad989. 2024-12-02T21:14:47,717 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ttt,1733174081291.d63da23e8236e18b416716ec838dd43e. 2024-12-02T21:14:47,717 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8e039115fcc0d0a54e2d03efed7e05de, disabling compactions & flushes 2024-12-02T21:14:47,717 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,iii,1733174081291.8e039115fcc0d0a54e2d03efed7e05de. 2024-12-02T21:14:47,717 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1733174081291.8e039115fcc0d0a54e2d03efed7e05de. 2024-12-02T21:14:47,717 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,iii,1733174081291.8e039115fcc0d0a54e2d03efed7e05de. after waiting 0 ms 2024-12-02T21:14:47,717 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,iii,1733174081291.8e039115fcc0d0a54e2d03efed7e05de. 2024-12-02T21:14:47,725 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/8e039115fcc0d0a54e2d03efed7e05de/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T21:14:47,725 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/a36d8f99c2aef77f88827585926ad989/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T21:14:47,725 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,zzz,1733174081291.a36d8f99c2aef77f88827585926ad989. 2024-12-02T21:14:47,725 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,iii,1733174081291.8e039115fcc0d0a54e2d03efed7e05de. 2024-12-02T21:14:47,725 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8e039115fcc0d0a54e2d03efed7e05de: Waiting for close lock at 1733174087717Running coprocessor pre-close hooks at 1733174087717Disabling compacts and flushes for region at 1733174087717Disabling writes for close at 1733174087717Writing region close event to WAL at 1733174087720 (+3 ms)Running coprocessor post-close hooks at 1733174087725 (+5 ms)Closed at 1733174087725 2024-12-02T21:14:47,725 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a36d8f99c2aef77f88827585926ad989: Waiting for close lock at 1733174087716Running coprocessor pre-close hooks at 1733174087716Disabling compacts and flushes for region at 1733174087716Disabling writes for close at 1733174087717 (+1 ms)Writing region close event to WAL at 1733174087720 (+3 ms)Running coprocessor post-close hooks at 1733174087725 (+5 ms)Closed at 1733174087725 2024-12-02T21:14:47,726 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,zzz,1733174081291.a36d8f99c2aef77f88827585926ad989. 2024-12-02T21:14:47,726 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,iii,1733174081291.8e039115fcc0d0a54e2d03efed7e05de. 2024-12-02T21:14:47,726 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0875ddf9cbd035a3345be0684b07b2e6, disabling compactions & flushes 2024-12-02T21:14:47,726 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 292fd0b4e0f07516cc25c069181e3111, disabling compactions & flushes 2024-12-02T21:14:47,726 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,yyy,1733174081291.0875ddf9cbd035a3345be0684b07b2e6. 2024-12-02T21:14:47,726 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,mmm,1733174081291.292fd0b4e0f07516cc25c069181e3111. 2024-12-02T21:14:47,726 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1733174081291.0875ddf9cbd035a3345be0684b07b2e6. 2024-12-02T21:14:47,726 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1733174081291.292fd0b4e0f07516cc25c069181e3111. 2024-12-02T21:14:47,726 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1733174081291.292fd0b4e0f07516cc25c069181e3111. after waiting 0 ms 2024-12-02T21:14:47,726 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1733174081291.0875ddf9cbd035a3345be0684b07b2e6. after waiting 0 ms 2024-12-02T21:14:47,726 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1733174081291.292fd0b4e0f07516cc25c069181e3111. 2024-12-02T21:14:47,726 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1733174081291.0875ddf9cbd035a3345be0684b07b2e6. 2024-12-02T21:14:47,729 DEBUG [RS_CLOSE_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/hbase/meta/1588230740/recovered.edits/78.seqid, newMaxSeqId=78, maxSeqId=64 2024-12-02T21:14:47,730 DEBUG [RS_CLOSE_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-02T21:14:47,730 INFO [RS_CLOSE_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-02T21:14:47,730 DEBUG [RS_CLOSE_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733174087660Running coprocessor pre-close hooks at 1733174087660Disabling compacts and flushes for region at 1733174087660Disabling writes for close at 1733174087660Obtaining lock to block concurrent updates at 1733174087660Preparing flush snapshotting stores in 1588230740 at 1733174087660Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=2720, getHeapSize=5504, getOffHeapSize=0, getCellsCount=24 at 1733174087668 (+8 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733174087669 (+1 ms)Flushing 1588230740/info: creating writer at 1733174087669Flushing 1588230740/info: appending metadata at 1733174087675 (+6 ms)Flushing 1588230740/info: closing flushed file at 1733174087675Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@66c2b175: reopening flushed file at 1733174087703 (+28 ms)Finished flush of dataSize ~2.66 KB/2720, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 56ms, sequenceid=75, compaction requested=false at 1733174087716 (+13 ms)Writing region close event to WAL at 1733174087723 (+7 ms)Running coprocessor post-close hooks at 1733174087730 (+7 ms)Closed at 1733174087730 2024-12-02T21:14:47,730 DEBUG [RS_CLOSE_META-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-02T21:14:47,731 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/292fd0b4e0f07516cc25c069181e3111/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T21:14:47,732 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/0875ddf9cbd035a3345be0684b07b2e6/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T21:14:47,732 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,mmm,1733174081291.292fd0b4e0f07516cc25c069181e3111. 2024-12-02T21:14:47,732 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 292fd0b4e0f07516cc25c069181e3111: Waiting for close lock at 1733174087726Running coprocessor pre-close hooks at 1733174087726Disabling compacts and flushes for region at 1733174087726Disabling writes for close at 1733174087726Writing region close event to WAL at 1733174087726Running coprocessor post-close hooks at 1733174087732 (+6 ms)Closed at 1733174087732 2024-12-02T21:14:47,732 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,mmm,1733174081291.292fd0b4e0f07516cc25c069181e3111. 2024-12-02T21:14:47,732 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,yyy,1733174081291.0875ddf9cbd035a3345be0684b07b2e6. 2024-12-02T21:14:47,732 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0875ddf9cbd035a3345be0684b07b2e6: Waiting for close lock at 1733174087726Running coprocessor pre-close hooks at 1733174087726Disabling compacts and flushes for region at 1733174087726Disabling writes for close at 1733174087726Writing region close event to WAL at 1733174087726Running coprocessor post-close hooks at 1733174087732 (+6 ms)Closed at 1733174087732 2024-12-02T21:14:47,732 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6a2fb6e9d39bf04774472d3b4c70810a, disabling compactions & flushes 2024-12-02T21:14:47,732 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,yyy,1733174081291.0875ddf9cbd035a3345be0684b07b2e6. 2024-12-02T21:14:47,732 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,vvv,1733174081291.6a2fb6e9d39bf04774472d3b4c70810a. 2024-12-02T21:14:47,733 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1733174081291.6a2fb6e9d39bf04774472d3b4c70810a. 2024-12-02T21:14:47,733 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1733174081291.6a2fb6e9d39bf04774472d3b4c70810a. after waiting 0 ms 2024-12-02T21:14:47,733 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 87ba365a3e5d028c9193155af96aeb60, disabling compactions & flushes 2024-12-02T21:14:47,733 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,nnn,1733174081291.87ba365a3e5d028c9193155af96aeb60. 2024-12-02T21:14:47,733 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1733174081291.6a2fb6e9d39bf04774472d3b4c70810a. 2024-12-02T21:14:47,733 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1733174081291.87ba365a3e5d028c9193155af96aeb60. 2024-12-02T21:14:47,733 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1733174081291.87ba365a3e5d028c9193155af96aeb60. after waiting 0 ms 2024-12-02T21:14:47,733 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1733174081291.87ba365a3e5d028c9193155af96aeb60. 2024-12-02T21:14:47,737 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/6a2fb6e9d39bf04774472d3b4c70810a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T21:14:47,737 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/87ba365a3e5d028c9193155af96aeb60/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T21:14:47,737 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,nnn,1733174081291.87ba365a3e5d028c9193155af96aeb60. 2024-12-02T21:14:47,737 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 87ba365a3e5d028c9193155af96aeb60: Waiting for close lock at 1733174087733Running coprocessor pre-close hooks at 1733174087733Disabling compacts and flushes for region at 1733174087733Disabling writes for close at 1733174087733Writing region close event to WAL at 1733174087733Running coprocessor post-close hooks at 1733174087737 (+4 ms)Closed at 1733174087737 2024-12-02T21:14:47,737 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,vvv,1733174081291.6a2fb6e9d39bf04774472d3b4c70810a. 2024-12-02T21:14:47,737 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6a2fb6e9d39bf04774472d3b4c70810a: Waiting for close lock at 1733174087732Running coprocessor pre-close hooks at 1733174087732Disabling compacts and flushes for region at 1733174087732Disabling writes for close at 1733174087733 (+1 ms)Writing region close event to WAL at 1733174087733Running coprocessor post-close hooks at 1733174087737 (+4 ms)Closed at 1733174087737 2024-12-02T21:14:47,737 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,nnn,1733174081291.87ba365a3e5d028c9193155af96aeb60. 2024-12-02T21:14:47,737 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,vvv,1733174081291.6a2fb6e9d39bf04774472d3b4c70810a. 2024-12-02T21:14:47,738 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 06d131e39cb4f5b02db4ec472d438a26, disabling compactions & flushes 2024-12-02T21:14:47,738 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,xxx,1733174081291.06d131e39cb4f5b02db4ec472d438a26. 2024-12-02T21:14:47,738 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 20545c8ee2d66091a91ff9697d1f18ca, disabling compactions & flushes 2024-12-02T21:14:47,738 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1733174081291.06d131e39cb4f5b02db4ec472d438a26. 2024-12-02T21:14:47,738 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ppp,1733174081291.20545c8ee2d66091a91ff9697d1f18ca. 2024-12-02T21:14:47,738 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1733174081291.06d131e39cb4f5b02db4ec472d438a26. after waiting 0 ms 2024-12-02T21:14:47,738 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1733174081291.20545c8ee2d66091a91ff9697d1f18ca. 2024-12-02T21:14:47,738 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1733174081291.06d131e39cb4f5b02db4ec472d438a26. 2024-12-02T21:14:47,738 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1733174081291.20545c8ee2d66091a91ff9697d1f18ca. after waiting 0 ms 2024-12-02T21:14:47,738 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1733174081291.20545c8ee2d66091a91ff9697d1f18ca. 2024-12-02T21:14:47,742 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/06d131e39cb4f5b02db4ec472d438a26/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T21:14:47,742 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/20545c8ee2d66091a91ff9697d1f18ca/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T21:14:47,743 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ppp,1733174081291.20545c8ee2d66091a91ff9697d1f18ca. 2024-12-02T21:14:47,743 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 20545c8ee2d66091a91ff9697d1f18ca: Waiting for close lock at 1733174087738Running coprocessor pre-close hooks at 1733174087738Disabling compacts and flushes for region at 1733174087738Disabling writes for close at 1733174087738Writing region close event to WAL at 1733174087738Running coprocessor post-close hooks at 1733174087743 (+5 ms)Closed at 1733174087743 2024-12-02T21:14:47,743 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ppp,1733174081291.20545c8ee2d66091a91ff9697d1f18ca. 2024-12-02T21:14:47,743 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,xxx,1733174081291.06d131e39cb4f5b02db4ec472d438a26. 2024-12-02T21:14:47,743 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 06d131e39cb4f5b02db4ec472d438a26: Waiting for close lock at 1733174087738Running coprocessor pre-close hooks at 1733174087738Disabling compacts and flushes for region at 1733174087738Disabling writes for close at 1733174087738Writing region close event to WAL at 1733174087738Running coprocessor post-close hooks at 1733174087743 (+5 ms)Closed at 1733174087743 2024-12-02T21:14:47,743 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,xxx,1733174081291.06d131e39cb4f5b02db4ec472d438a26. 2024-12-02T21:14:47,743 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0dfd4169e4c254c51bcc70a472f99ecf, disabling compactions & flushes 2024-12-02T21:14:47,743 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ggg,1733174081291.0dfd4169e4c254c51bcc70a472f99ecf. 2024-12-02T21:14:47,743 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1733174081291.0dfd4169e4c254c51bcc70a472f99ecf. 2024-12-02T21:14:47,743 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1733174081291.0dfd4169e4c254c51bcc70a472f99ecf. after waiting 0 ms 2024-12-02T21:14:47,743 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1733174081291.0dfd4169e4c254c51bcc70a472f99ecf. 2024-12-02T21:14:47,749 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/0dfd4169e4c254c51bcc70a472f99ecf/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T21:14:47,750 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ggg,1733174081291.0dfd4169e4c254c51bcc70a472f99ecf. 2024-12-02T21:14:47,750 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0dfd4169e4c254c51bcc70a472f99ecf: Waiting for close lock at 1733174087743Running coprocessor pre-close hooks at 1733174087743Disabling compacts and flushes for region at 1733174087743Disabling writes for close at 1733174087743Writing region close event to WAL at 1733174087744 (+1 ms)Running coprocessor post-close hooks at 1733174087750 (+6 ms)Closed at 1733174087750 2024-12-02T21:14:47,750 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ggg,1733174081291.0dfd4169e4c254c51bcc70a472f99ecf. 2024-12-02T21:14:47,750 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5d60c566b39516dd875aaf2bd7def402, disabling compactions & flushes 2024-12-02T21:14:47,751 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,jjj,1733174081291.5d60c566b39516dd875aaf2bd7def402. 2024-12-02T21:14:47,751 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1733174081291.5d60c566b39516dd875aaf2bd7def402. 2024-12-02T21:14:47,751 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1733174081291.5d60c566b39516dd875aaf2bd7def402. after waiting 0 ms 2024-12-02T21:14:47,751 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1733174081291.5d60c566b39516dd875aaf2bd7def402. 2024-12-02T21:14:47,755 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/5d60c566b39516dd875aaf2bd7def402/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-02T21:14:47,756 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,jjj,1733174081291.5d60c566b39516dd875aaf2bd7def402. 2024-12-02T21:14:47,756 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5d60c566b39516dd875aaf2bd7def402: Waiting for close lock at 1733174087750Running coprocessor pre-close hooks at 1733174087750Disabling compacts and flushes for region at 1733174087750Disabling writes for close at 1733174087751 (+1 ms)Writing region close event to WAL at 1733174087751Running coprocessor post-close hooks at 1733174087756 (+5 ms)Closed at 1733174087756 2024-12-02T21:14:47,756 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,jjj,1733174081291.5d60c566b39516dd875aaf2bd7def402. 2024-12-02T21:14:47,756 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 120cf3776e92a15d3276f11911c4042f, disabling compactions & flushes 2024-12-02T21:14:47,756 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,eee,1733174081291.120cf3776e92a15d3276f11911c4042f. 2024-12-02T21:14:47,756 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1733174081291.120cf3776e92a15d3276f11911c4042f. 2024-12-02T21:14:47,756 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,eee,1733174081291.120cf3776e92a15d3276f11911c4042f. after waiting 0 ms 2024-12-02T21:14:47,756 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,eee,1733174081291.120cf3776e92a15d3276f11911c4042f. 2024-12-02T21:14:47,760 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/data/default/test/120cf3776e92a15d3276f11911c4042f/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-12-02T21:14:47,761 INFO [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,eee,1733174081291.120cf3776e92a15d3276f11911c4042f. 2024-12-02T21:14:47,761 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 120cf3776e92a15d3276f11911c4042f: Waiting for close lock at 1733174087756Running coprocessor pre-close hooks at 1733174087756Disabling compacts and flushes for region at 1733174087756Disabling writes for close at 1733174087756Writing region close event to WAL at 1733174087756Running coprocessor post-close hooks at 1733174087761 (+5 ms)Closed at 1733174087761 2024-12-02T21:14:47,761 DEBUG [RS_CLOSE_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,eee,1733174081291.120cf3776e92a15d3276f11911c4042f. 2024-12-02T21:14:47,857 WARN [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,lll,1733174081291.0d0be65f6ffd7ebb4c8b2fee37042136., aborting... java.io.IOException: Failed to report opened region to master: test,lll,1733174081291.0d0be65f6ffd7ebb4c8b2fee37042136. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:14:47,858 ERROR [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2440): ***** ABORTING region server 87c3fdb6c570,40413,1733174078061: Failed to open region test,lll,1733174081291.0d0be65f6ffd7ebb4c8b2fee37042136. and can not recover ***** java.io.IOException: Failed to report opened region to master: test,lll,1733174081291.0d0be65f6ffd7ebb4c8b2fee37042136. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T21:14:47,858 ERROR [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-02T21:14:47,858 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-02T21:14:47,859 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-02T21:14:47,859 INFO [RS:1;87c3fdb6c570:36585 {}] regionserver.HRegionServer(976): stopping server 87c3fdb6c570,36585,1733174078182; all regions closed. 2024-12-02T21:14:47,860 INFO [RS:0;87c3fdb6c570:40413 {}] regionserver.HRegionServer(976): stopping server 87c3fdb6c570,40413,1733174078061; all regions closed. 2024-12-02T21:14:47,861 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-02T21:14:47,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42827 is added to blk_1073741865_1041 (size=4204) 2024-12-02T21:14:47,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741865_1041 (size=4204) 2024-12-02T21:14:47,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36203 is added to blk_1073741865_1041 (size=4204) 2024-12-02T21:14:47,865 DEBUG [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-02T21:14:47,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741833_1009 (size=7064) 2024-12-02T21:14:47,866 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/WALs/87c3fdb6c570,36585,1733174078182/87c3fdb6c570%2C36585%2C1733174078182.1733174080466 not finished, retry = 0 2024-12-02T21:14:47,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42827 is added to blk_1073741833_1009 (size=7064) 2024-12-02T21:14:47,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36203 is added to blk_1073741833_1009 (size=7064) 2024-12-02T21:14:47,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741834_1010 (size=5194) 2024-12-02T21:14:47,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36203 is added to blk_1073741834_1010 (size=5194) 2024-12-02T21:14:47,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42827 is added to blk_1073741834_1010 (size=5194) 2024-12-02T21:14:47,870 DEBUG [RS:0;87c3fdb6c570:40413 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:14:47,870 INFO [RS:0;87c3fdb6c570:40413 {}] regionserver.LeaseManager(133): Closed leases 2024-12-02T21:14:47,870 INFO [RS:0;87c3fdb6c570:40413 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-02T21:14:47,871 INFO [RS:0;87c3fdb6c570:40413 {}] hbase.ChoreService(370): Chore service for: regionserver/87c3fdb6c570:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-02T21:14:47,871 INFO [RS:0;87c3fdb6c570:40413 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-02T21:14:47,872 INFO [regionserver/87c3fdb6c570:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-02T21:14:47,873 INFO [RS_OPEN_REGION-regionserver/87c3fdb6c570:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 514426112 }, "NonHeapMemoryUsage": { "committed": 164495360, "init": 7667712, "max": -1, "used": 161941528 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "87c3fdb6c570", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 2, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 36078, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 105, "ProcessCallTime_min": 0, "ProcessCallTime_max": 14, "ProcessCallTime_mean": 2, "ProcessCallTime_25th_percentile": 3, "ProcessCallTime_median": 7, "ProcessCallTime_75th_percentile": 10, "ProcessCallTime_90th_percentile": 12, "ProcessCallTime_95th_percentile": 13, "ProcessCallTime_98th_percentile": 13, "ProcessCallTime_99th_percentile": 13, "ProcessCallTime_99.9th_percentile": 13, "ProcessCallTime_TimeRangeCount_0-1": 105, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 105, "QueueCallTime_min": 0, "QueueCallTime_max": 15, "QueueCallTime_mean": 1, "QueueCallTime_25th_percentile": 3, "QueueCallTime_median": 7, "QueueCallTime_75th_percentile": 11, "QueueCallTime_90th_percentile": 13, "QueueCallTime_95th_percentile": 14, "QueueCallTime_98th_percentile": 14, "QueueCallTime_99th_percentile": 14, "QueueCallTime_99.9th_percentile": 14, "QueueCallTime_TimeRangeCount_0-1": 105, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 105, "TotalCallTime_min": 0, "TotalCallTime_max": 17, "TotalCallTime_mean": 3, "TotalCallTime_25th_percentile": 4, "TotalCallTime_median": 8, "TotalCallTime_75th_percentile": 12, "TotalCallTime_90th_percentile": 15, "TotalCallTime_95th_percentile": 16, "TotalCallTime_98th_percentile": 16, "TotalCallTime_99th_percentile": 16, "TotalCallTime_99.9th_percentile": 16, "TotalCallTime_TimeRangeCount_0-1": 105, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 105, "ResponseSize_min": 0, "ResponseSize_max": 22359, "ResponseSize_mean": 651, "ResponseSize_25th_percentile": 5589, "ResponseSize_median": 11179, "ResponseSize_75th_percentile": 16769, "ResponseSize_90th_percentile": 20123, "ResponseSize_95th_percentile": 21241, "ResponseSize_98th_percentile": 21911, "ResponseSize_99th_percentile": 22135, "ResponseSize_99.9th_percentile": 22336, "ResponseSize_SizeRangeCount_0-10": 105, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 105, "RequestSize_min": 81, "RequestSize_max": 8095, "RequestSize_mean": 312, "RequestSize_25th_percentile": 2084, "RequestSize_median": 4088, "RequestSize_75th_percentile": 6091, "RequestSize_90th_percentile": 7293, "RequestSize_95th_percentile": 7694, "RequestSize_98th_percentile": 7934, "RequestSize_99th_percentile": 8014, "RequestSize_99.9th_percentile": 8086, "RequestSize_SizeRangeCount_0-10": 105, "sentBytes": 54797 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "87c3fdb6c570", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:63400", "tag.serverName": "87c3fdb6c570,33459,1733174078298", "tag.clusterId": "a3fb6e34-ca27-4453-9e72-69b912f26690", "tag.Context": "regionserver", "tag.Hostname": "87c3fdb6c570", "regionCount": 9, "storeCount": 12, "hlogFileCount": 2, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 30111, "memStoreHeapSize": 53936, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1733174078298, "averageRegionSize": 3345, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 3341, "localBytesRead": 3341, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 18.8, "writeRequestRatePerSecond": 17.2, "ByteBuffAllocatorHeapAllocationBytes": 7474, "ByteBuffAllocatorPoolAllocationBytes": 465920, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 79, "totalRowActionRequestCount": 180, "readRequestCount": 94, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 86, "rpcGetRequestCount": 2, "rpcFullScanRequestCount": 9, "rpcScanRequestCount": 16, "rpcMultiRequestCount": 1, "rpcMutateRequestCount": 60, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 16, "ScanTime_min": 0, "ScanTime_max": 10, "ScanTime_mean": 1, "ScanTime_25th_percentile": 2, "ScanTime_median": 5, "ScanTime_75th_percentile": 7, "ScanTime_90th_percentile": 9, "ScanTime_95th_percentile": 9, "ScanTime_98th_percentile": 9, "ScanTime_99th_percentile": 9, "ScanTime_99.9th_percentile": 9, "ScanTime_TimeRangeCount_0-1": 16, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 60, "Put_min": 1, "Put_max": 7, "Put_mean": 2, "Put_25th_percentile": 2, "Put_median": 4, "Put_75th_percentile": 5, "Put_90th_percentile": 6, "Put_95th_percentile": 6, "Put_98th_percentile": 6, "Put_99th_percentile": 6, "Put_99.9th_percentile": 6, "Put_TimeRangeCount_0-1": 60, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 1, "PutBatch_min": 8, "PutBatch_max": 8, "PutBatch_mean": 8, "PutBatch_25th_percentile": 8, "PutBatch_median": 8, "PutBatch_75th_percentile": 8, "PutBatch_90th_percentile": 8, "PutBatch_95th_percentile": 8, "PutBatch_98th_percentile": 8, "PutBatch_99th_percentile": 8, "PutBatch_99.9th_percentile": 8, "PutBatch_TimeRangeCount_0-1": 1, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 2, "Get_min": 1, "Get_max": 1, "Get_mean": 1, "Get_25th_percentile": 1, "Get_median": 1, "Get_75th_percentile": 1, "Get_90th_percentile": 1, "Get_95th_percentile": 1, "Get_98th_percentile": 1, "Get_99th_percentile": 1, "Get_99.9th_percentile": 1, "Get_TimeRangeCount_0-1": 2, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 16, "ScanSize_min": 0, "ScanSize_max": 22232, "ScanSize_mean": 4186, "ScanSize_25th_percentile": 5558, "ScanSize_median": 11116, "ScanSize_75th_percentile": 16674, "ScanSize_90th_percentile": 20008, "ScanSize_95th_percentile": 21120, "ScanSize_98th_percentile": 21787, "ScanSize_99th_percentile": 22009, "ScanSize_99.9th_percentile": 22209, "ScanSize_SizeRangeCount_0-10": 16, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-12-02T21:14:47,874 INFO [RS:0;87c3fdb6c570:40413 {}] executor.ExecutorService(108): Executor-11-RS_OPEN_REGION-regionserver/87c3fdb6c570:0 had [Event #38 of type M_RS_OPEN_REGION (class org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler), Event #39 of type M_RS_OPEN_REGION (class org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler), Event #40 of type M_RS_OPEN_REGION (class org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler)] on shutdown 2024-12-02T21:14:47,874 INFO [RS:0;87c3fdb6c570:40413 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:40413 2024-12-02T21:14:47,887 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42045-0x10197e5a0060000, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-02T21:14:47,887 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40413-0x10197e5a0060001, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/87c3fdb6c570,40413,1733174078061 2024-12-02T21:14:47,887 INFO [RS:0;87c3fdb6c570:40413 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-02T21:14:47,897 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [87c3fdb6c570,40413,1733174078061] 2024-12-02T21:14:47,907 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/87c3fdb6c570,40413,1733174078061 already deleted, retry=false 2024-12-02T21:14:47,908 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 87c3fdb6c570,40413,1733174078061 expired; onlineServers=1 2024-12-02T21:14:47,972 DEBUG [RS:1;87c3fdb6c570:36585 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/oldWALs 2024-12-02T21:14:47,972 INFO [RS:1;87c3fdb6c570:36585 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 87c3fdb6c570%2C36585%2C1733174078182:(num 1733174080466) 2024-12-02T21:14:47,972 DEBUG [RS:1;87c3fdb6c570:36585 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T21:14:47,972 INFO [RS:1;87c3fdb6c570:36585 {}] regionserver.LeaseManager(133): Closed leases 2024-12-02T21:14:47,972 INFO [RS:1;87c3fdb6c570:36585 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-02T21:14:47,973 INFO [RS:1;87c3fdb6c570:36585 {}] hbase.ChoreService(370): Chore service for: regionserver/87c3fdb6c570:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-02T21:14:47,973 INFO [RS:1;87c3fdb6c570:36585 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-02T21:14:47,973 INFO [regionserver/87c3fdb6c570:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-02T21:14:47,973 INFO [RS:1;87c3fdb6c570:36585 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-02T21:14:47,973 INFO [RS:1;87c3fdb6c570:36585 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-02T21:14:47,973 INFO [RS:1;87c3fdb6c570:36585 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-02T21:14:47,973 INFO [RS:1;87c3fdb6c570:36585 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:36585 2024-12-02T21:14:47,981 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42045-0x10197e5a0060000, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-02T21:14:47,981 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36585-0x10197e5a0060002, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/87c3fdb6c570,36585,1733174078182 2024-12-02T21:14:47,981 INFO [RS:1;87c3fdb6c570:36585 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-02T21:14:47,992 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [87c3fdb6c570,36585,1733174078182] 2024-12-02T21:14:47,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40413-0x10197e5a0060001, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T21:14:47,997 INFO [RS:0;87c3fdb6c570:40413 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-02T21:14:47,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40413-0x10197e5a0060001, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T21:14:47,997 INFO [RS:0;87c3fdb6c570:40413 {}] regionserver.HRegionServer(1031): Exiting; stopping=87c3fdb6c570,40413,1733174078061; zookeeper connection closed. 2024-12-02T21:14:47,998 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1db4a7 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1db4a7 2024-12-02T21:14:48,002 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/87c3fdb6c570,36585,1733174078182 already deleted, retry=false 2024-12-02T21:14:48,002 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 87c3fdb6c570,36585,1733174078182 expired; onlineServers=0 2024-12-02T21:14:48,002 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '87c3fdb6c570,42045,1733174077295' ***** 2024-12-02T21:14:48,002 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-02T21:14:48,003 INFO [M:0;87c3fdb6c570:42045 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-02T21:14:48,003 INFO [M:0;87c3fdb6c570:42045 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-02T21:14:48,003 DEBUG [M:0;87c3fdb6c570:42045 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-02T21:14:48,003 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-02T21:14:48,003 DEBUG [M:0;87c3fdb6c570:42045 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-02T21:14:48,003 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster-HFileCleaner.large.0-1733174080064 {}] cleaner.HFileCleaner(306): Exit Thread[master/87c3fdb6c570:0:becomeActiveMaster-HFileCleaner.large.0-1733174080064,5,FailOnTimeoutGroup] 2024-12-02T21:14:48,003 DEBUG [master/87c3fdb6c570:0:becomeActiveMaster-HFileCleaner.small.0-1733174080065 {}] cleaner.HFileCleaner(306): Exit Thread[master/87c3fdb6c570:0:becomeActiveMaster-HFileCleaner.small.0-1733174080065,5,FailOnTimeoutGroup] 2024-12-02T21:14:48,003 INFO [M:0;87c3fdb6c570:42045 {}] hbase.ChoreService(370): Chore service for: master/87c3fdb6c570:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-02T21:14:48,003 INFO [M:0;87c3fdb6c570:42045 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-02T21:14:48,003 DEBUG [M:0;87c3fdb6c570:42045 {}] master.HMaster(1795): Stopping service threads 2024-12-02T21:14:48,003 INFO [M:0;87c3fdb6c570:42045 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-02T21:14:48,004 INFO [M:0;87c3fdb6c570:42045 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T21:14:48,004 INFO [M:0;87c3fdb6c570:42045 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-02T21:14:48,004 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-02T21:14:48,013 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42045-0x10197e5a0060000, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-02T21:14:48,013 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42045-0x10197e5a0060000, quorum=127.0.0.1:63400, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T21:14:48,013 DEBUG [M:0;87c3fdb6c570:42045 {}] zookeeper.ZKUtil(347): master:42045-0x10197e5a0060000, quorum=127.0.0.1:63400, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-02T21:14:48,013 WARN [M:0;87c3fdb6c570:42045 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-02T21:14:48,014 INFO [M:0;87c3fdb6c570:42045 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/.lastflushedseqids 2024-12-02T21:14:48,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36203 is added to blk_1073741870_1046 (size=1) 2024-12-02T21:14:48,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42827 is added to blk_1073741870_1046 (size=1) 2024-12-02T21:14:48,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741870_1046 (size=1) 2024-12-02T21:14:48,028 INFO [M:0;87c3fdb6c570:42045 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-02T21:14:48,028 INFO [M:0;87c3fdb6c570:42045 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-02T21:14:48,028 DEBUG [M:0;87c3fdb6c570:42045 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-02T21:14:48,028 INFO [M:0;87c3fdb6c570:42045 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T21:14:48,028 DEBUG [M:0;87c3fdb6c570:42045 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T21:14:48,028 DEBUG [M:0;87c3fdb6c570:42045 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-02T21:14:48,028 DEBUG [M:0;87c3fdb6c570:42045 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T21:14:48,028 INFO [M:0;87c3fdb6c570:42045 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=259.53 KB heapSize=315.44 KB 2024-12-02T21:14:48,048 DEBUG [M:0;87c3fdb6c570:42045 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a91c28c0f1d946578981144f9b65bdc7 is 82, key is hbase:meta,,1/info:regioninfo/1733174086121/Put/seqid=0 2024-12-02T21:14:48,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40959 is added to blk_1073741871_1047 (size=6071) 2024-12-02T21:14:48,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42827 is added to blk_1073741871_1047 (size=6071) 2024-12-02T21:14:48,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36203 is added to blk_1073741871_1047 (size=6071) 2024-12-02T21:14:48,056 INFO [M:0;87c3fdb6c570:42045 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=599 (bloomFilter=true), to=hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a91c28c0f1d946578981144f9b65bdc7 2024-12-02T21:14:48,082 DEBUG [M:0;87c3fdb6c570:42045 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43251/user/jenkins/test-data/337ece9e-e2a3-4a72-093b-eedca134d70b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/63084aa0ddfe40d5a1ced9a6556ad06f is 2990, key is \x00\x00\x00\x00\x00\x00\x00